From e5555b16cd1b00876d862dd077463fb9f0aec39a Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 16 Apr 2024 14:37:53 +0800 Subject: [PATCH 01/20] feat: add GetCollectionNameAsync error log --- .../Sharding/CollectionRouteKeyProvider.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs index 60e595ed..8ba06fd4 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs @@ -111,14 +111,27 @@ public async Task> GetCollectionNameAsync(List(s => s.Index(collectionRouteKeyIndexName).Size(10000).Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) .Collapse(c => c.Field(f=>f.CollectionName)).Aggregations(a => a .Cardinality("courseAgg", ca => ca.Field(f=>f.CollectionName)))); + if (result == null) + { + _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result is null"); + } if (!result.IsValid) { throw new ElasticsearchException($"Search document failed at index {collectionRouteKeyIndexName} :" + result.ServerError.Error.Reason); } + + if (result.Documents == null) + { + _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result.Documents is null"); + } var collectionList = result.Documents.ToList(); _logger.LogDebug($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: " + $"collectionList: {JsonConvert.SerializeObject(collectionList)}"); From e723fe04072649ac1d0335f276f7cbf8284c9afa Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 16 Apr 2024 14:41:29 +0800 Subject: [PATCH 02/20] feat: optimize log --- .../Sharding/CollectionRouteKeyProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs index 8ba06fd4..44b83132 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs @@ -121,7 +121,7 @@ public async Task> GetCollectionNameAsync(List ca.Field(f=>f.CollectionName)))); if (result == null) { - _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result is null"); + _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result is null fieldValue:{fieldValue}"); } if (!result.IsValid) { @@ -130,7 +130,7 @@ public async Task> GetCollectionNameAsync(List Date: Tue, 16 Apr 2024 16:06:57 +0800 Subject: [PATCH 03/20] feat: add route key provider log --- .../Sharding/CollectionRouteKeyProvider.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs index 44b83132..060a35d4 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs @@ -110,6 +110,10 @@ public async Task> GetCollectionNameAsync(List> GetCollectionNameAsync(List Date: Tue, 16 Apr 2024 21:03:42 +0800 Subject: [PATCH 04/20] feat: optimize get route key collection search --- .../Sharding/CollectionRouteKeyProvider.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs index 060a35d4..992cdd92 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs @@ -119,10 +119,15 @@ public async Task> GetCollectionNameAsync(List(s => + // s.Index(collectionRouteKeyIndexName).Size(10000).Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) + // .Collapse(c => c.Field(f=>f.CollectionName)).Aggregations(a => a + // .Cardinality("courseAgg", ca => ca.Field(f=>f.CollectionName)))); var result = await client.SearchAsync(s => - s.Index(collectionRouteKeyIndexName).Size(10000).Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) - .Collapse(c => c.Field(f=>f.CollectionName)).Aggregations(a => a - .Cardinality("courseAgg", ca => ca.Field(f=>f.CollectionName)))); + s.Index(collectionRouteKeyIndexName) + .Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) + .Collapse(c => c.Field(f => f.CollectionName)) + .Size(1000)); if (result == null) { _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result is null fieldValue:{fieldValue}"); From 49dce54b49af8065fcbf8697b2c4c4b74ed64e28 Mon Sep 17 00:00:00 2001 From: "louis.li" Date: Thu, 18 Apr 2024 13:37:02 +0800 Subject: [PATCH 05/20] feat:test Sub object Query --- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 15 ++++++++++++++ .../Linq/ElasticsearchQueryExecutor.cs | 20 +++++++++++++++++-- .../Linq/GeneratorExpressionTreeVisitor.cs | 18 ++++++++++++++++- .../Linq/QueryModelExtensions.cs | 20 +++++++++++++++++-- .../Entities/BlockIndex.cs | 8 ++++++++ .../ElasticsearchRepositoryTests.cs | 15 +++++++++++--- 6 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index 7f8db7a5..d4dc4c51 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -110,6 +110,21 @@ protected override void VisitResultOperators(ObservableCollection(QueryModel queryModel) { descriptor.Query(q => queryAggregator.Query); } - // var dsl = _elasticClient.RequestResponseSerializer.SerializeToString(descriptor); + var dsl = _elasticClient.RequestResponseSerializer.SerializeToString(descriptor); return descriptor; }); if (!response.IsValid) @@ -312,6 +312,22 @@ private string GetIndexName(QueryModel queryModel) AsyncHelper.RunSync(async () => await _collectionNameProvider.GetFullCollectionNameAsync(conditions)); return IndexNameHelper.FormatIndexName(indexNames); } + + private static string GetCollectionNameKey(MemberExpression memberExpression) + { + string key = memberExpression.Member.Name; + while (memberExpression.Expression != null) + { + memberExpression = memberExpression.Expression as MemberExpression; + if (memberExpression == null) + { + break; + } + key = memberExpression.Member.Name +"."+ key; + return key; + } + return key; + } } public class Grouping : IGrouping @@ -336,4 +352,4 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } } -} \ No newline at end of file +} diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 940712e2..28835392 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -126,13 +126,29 @@ protected override Expression VisitMethodCall(MethodCallExpression expression) return expression; } + + private static string GetFullNameKey(MemberExpression memberExpression) + { + string key = memberExpression.Member.Name; + while (memberExpression.Expression != null) + { + memberExpression = memberExpression.Expression as MemberExpression; + if (memberExpression == null) + { + break; + } + key = memberExpression.Member.Name +"."+ key; + return key; + } + return key; + } protected override Expression VisitMember(MemberExpression expression) { Visit(expression.Expression); PropertyType = Nullable.GetUnderlyingType(expression.Type) ?? expression.Type; - PropertyName = _propertyNameInferrerParser.Parser(expression.Member.Name); + PropertyName = _propertyNameInferrerParser.Parser(GetFullNameKey(expression)); // Implicit boolean is only a member visit if (expression.Type == typeof(bool)) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs index 3ff88165..84128725 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs @@ -56,13 +56,14 @@ private static void VisitBinaryExpression(List conditio { var memberExpression = expression.Left as MemberExpression; var constantExpression = expression.Right as ConstantExpression; - conditions.Add(new CollectionNameCondition { - Key = memberExpression.Member.Name, + + Key = GetCollectionNameKey(memberExpression), Value = constantExpression.Value, Type = GetConditionType(expression.NodeType) }); + return; } @@ -105,4 +106,19 @@ private static ConditionType GetConditionType(ExpressionType expressionType) throw new ArgumentOutOfRangeException(nameof(expressionType), expressionType, null); } } + private static string GetCollectionNameKey(MemberExpression memberExpression) + { + string key = memberExpression.Member.Name; + while (memberExpression.Expression != null) + { + memberExpression = memberExpression.Expression as MemberExpression; + if (memberExpression == null) + { + break; + } + key = memberExpression.Member.Name +"."+ key; + return key; + } + return key; + } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs index cd28db54..5ede5cd7 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs @@ -1,4 +1,5 @@ using AElf.EntityMapping.Entities; +using Nest; namespace AElf.EntityMapping.Elasticsearch.Entities; @@ -6,4 +7,11 @@ public class BlockIndex:BlockBase,IEntityMappingEntity { public List TransactionIds { get; set; } public int LogEventCount { get; set; } + [Keyword]public string TxnFee { get; set; } + public FeeIndex Fee { get; set; } +} + +public class FeeIndex +{ + [Keyword] public string BlockFee { get; set; } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index 8a876193..3adf29e5 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -563,7 +563,12 @@ public async Task GetList_MultipleChain_Test() BlockHeight = i, BlockTime = DateTime.Now.AddDays(-10 + i), LogEventCount = i, - ChainId = "AELF" + ChainId = "AELF", + TxnFee = "BlockHash" + i, + Fee = new FeeIndex() + { + BlockFee = "BlockHash" + i, + } }; await _elasticsearchRepository.AddAsync(blockIndex); } @@ -582,10 +587,14 @@ public async Task GetList_MultipleChain_Test() await _elasticsearchRepository.AddAsync(blockIndex); } - var list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="AELF" && o.BlockHeight >= 0); + /*var list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="AELF" && o.BlockHeight >= 0); list.Count.ShouldBe(7); list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="tDVV" && o.BlockHeight >= 0); - list.Count.ShouldBe(11); + list.Count.ShouldBe(11);*/ + + var list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="AELF" && o.BlockHeight >= 0 && o.Fee.BlockFee == "BlockHash2" ); + list.Count.ShouldBe(1); } + } \ No newline at end of file From 93f344c1dc0d60b333082f02977366675c975124 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 18 Apr 2024 21:21:23 +0800 Subject: [PATCH 06/20] fix: query with multi field --- .../Linq/GeneratorExpressionTreeVisitor.cs | 21 +-- .../Linq/QueryModelExtensions.cs | 2 +- .../Services/EnsureIndexBuildService.cs | 22 ++- .../AElf.EntityMapping.Tests.csproj | 1 + .../AElfEntityMappingTestModule.cs | 145 ++++++++++-------- 5 files changed, 100 insertions(+), 91 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 28835392..9811921d 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -126,10 +126,10 @@ protected override Expression VisitMethodCall(MethodCallExpression expression) return expression; } - - private static string GetFullNameKey(MemberExpression memberExpression) + + private string GetFullNameKey(MemberExpression memberExpression) { - string key = memberExpression.Member.Name; + var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); while (memberExpression.Expression != null) { memberExpression = memberExpression.Expression as MemberExpression; @@ -137,9 +137,11 @@ private static string GetFullNameKey(MemberExpression memberExpression) { break; } - key = memberExpression.Member.Name +"."+ key; + + key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); return key; } + return key; } @@ -204,9 +206,9 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) { Visit(whereClause.Predicate); Node tmp = (Node)QueryMap[whereClause.Predicate].Clone(); - QueryMap[expression] = tmp ; + QueryMap[expression] = tmp; QueryMap[expression].IsSubQuery = true; - QueryMap[expression].SubQueryPath = from;//from.ToLower(); + QueryMap[expression].SubQueryPath = from; //from.ToLower(); QueryMap[expression].SubQueryFullPath = fullPath; // VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; @@ -214,6 +216,7 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) { VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true); } + if (predicate.Right is BinaryExpression) { VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true); @@ -396,10 +399,10 @@ private Node HandleStringProperty(Expression expression) { case ExpressionType.Equal: return new TermNode(PropertyName, Value); - // return new MatchPhraseNode(PropertyName, Value); + // return new MatchPhraseNode(PropertyName, Value); case ExpressionType.NotEqual: return new NotNode(new TermNode(PropertyName, Value)); - //return new NotNode(new MatchPhraseNode(PropertyName, Value)); + //return new NotNode(new MatchPhraseNode(PropertyName, Value)); default: return null; } @@ -572,9 +575,9 @@ private object ConvertEnumValue(Type entityType, string propertyName, object val return (int)enumValue; } + protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, bool parentIsSubQuery) { - if (expression.Left is BinaryExpression && expression.Right is ConstantExpression) { return; diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs index 84128725..57c58f6f 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs @@ -108,7 +108,7 @@ private static ConditionType GetConditionType(ExpressionType expressionType) } private static string GetCollectionNameKey(MemberExpression memberExpression) { - string key = memberExpression.Member.Name; + var key = memberExpression.Member.Name; while (memberExpression.Expression != null) { memberExpression = memberExpression.Expression as MemberExpression; diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs index 3b4fe710..5bfdf065 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs @@ -10,14 +10,14 @@ namespace AElf.EntityMapping.Elasticsearch.Services; -public class EnsureIndexBuildService: IEnsureIndexBuildService, ITransientDependency +public class EnsureIndexBuildService : IEnsureIndexBuildService, ITransientDependency { private readonly IElasticIndexService _elasticIndexService; private readonly List _modules; private readonly ElasticsearchOptions _elasticsearchOptions; private readonly AElfEntityMappingOptions _entityMappingOptions; - - + + public EnsureIndexBuildService(IOptions moduleConfiguration, IElasticIndexService elasticIndexService, IOptions entityMappingOptions, @@ -28,7 +28,7 @@ public EnsureIndexBuildService(IOptions moduleConfigura _elasticsearchOptions = elasticsearchOptions.Value; _entityMappingOptions = entityMappingOptions.Value; } - + public void EnsureIndexesCreate() { AsyncHelper.RunSync(async () => @@ -39,19 +39,19 @@ public void EnsureIndexesCreate() } }); } - + private async Task HandleModuleAsync(Type moduleType) { var types = GetTypesAssignableFrom(moduleType.Assembly); foreach (var t in types) { - var indexName = IndexNameHelper.GetDefaultFullIndexName(t,_entityMappingOptions.CollectionPrefix); - + var indexName = IndexNameHelper.GetDefaultFullIndexName(t, _entityMappingOptions.CollectionPrefix); + if (IsShardingCollection(t)) { //if shard index, create index Template var indexTemplateName = indexName + "-template"; - await _elasticIndexService.CreateIndexTemplateAsync(indexTemplateName,indexName, t, + await _elasticIndexService.CreateIndexTemplateAsync(indexTemplateName, indexName, t, _elasticsearchOptions.NumberOfShards, _elasticsearchOptions.NumberOfReplicas); //create index marked field cache @@ -66,9 +66,7 @@ await _elasticIndexService.CreateCollectionRouteKeyIndexAsync(t, _elasticsearchO await _elasticIndexService.CreateIndexAsync(indexName, t, _elasticsearchOptions.NumberOfShards, _elasticsearchOptions.NumberOfReplicas); } - } - } private async Task CreateShardingCollectionTailIndexAsync() @@ -86,7 +84,7 @@ private List GetTypesAssignableFrom(Assembly assembly) !type.IsAbstract && type.IsClass && compareType != type) .Cast().ToList(); } - + private bool IsShardingCollection(Type type) { if (_entityMappingOptions == null || _entityMappingOptions.ShardInitSettings == null) @@ -94,6 +92,4 @@ private bool IsShardingCollection(Type type) var options = _entityMappingOptions.ShardInitSettings.Find(a => a.CollectionName == type.Name); return options != null; } - - } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Tests/AElf.EntityMapping.Tests.csproj b/test/AElf.EntityMapping.Tests/AElf.EntityMapping.Tests.csproj index c9ca4a06..7efad299 100644 --- a/test/AElf.EntityMapping.Tests/AElf.EntityMapping.Tests.csproj +++ b/test/AElf.EntityMapping.Tests/AElf.EntityMapping.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/test/AElf.EntityMapping.Tests/AElfEntityMappingTestModule.cs b/test/AElf.EntityMapping.Tests/AElfEntityMappingTestModule.cs index a3d01f2d..0897b43c 100644 --- a/test/AElf.EntityMapping.Tests/AElfEntityMappingTestModule.cs +++ b/test/AElf.EntityMapping.Tests/AElfEntityMappingTestModule.cs @@ -1,3 +1,4 @@ +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Options; using AElf.EntityMapping.Sharding; using AElf.EntityMapping.TestBase; @@ -20,6 +21,10 @@ public override void ConfigureServices(ServiceConfigurationContext context) { x.AddModule(typeof(AElfEntityMappingTestModule)); }); + Configure(x => + { + x.Uris = new List {"http://localhost:9200"}; + }); context.Services.Configure(options => { @@ -31,93 +36,97 @@ public override void ConfigureServices(ServiceConfigurationContext context) private List InitShardInitSettingOptions() { - ShardInitSetting blockIndexDto = new ShardInitSetting(); - blockIndexDto.CollectionName = "BlockIndex"; - blockIndexDto.ShardGroups = new List() + var blockIndexDto = new ShardInitSetting { - new ShardGroup() + CollectionName = "BlockIndex", + ShardGroups = new List() { - ShardKeys = new List() + new ShardGroup() { - new ShardKey() - { - Name = "ChainId", - Value = "AELF", - Step = "", - StepType = StepType.None - }, - new ShardKey() + ShardKeys = new List() { - Name = "BlockHeight", - Value = "0", - Step = "5", - StepType = StepType.Floor + new ShardKey() + { + Name = "ChainId", + Value = "AELF", + Step = "", + StepType = StepType.None + }, + new ShardKey() + { + Name = "BlockHeight", + Value = "0", + Step = "5", + StepType = StepType.Floor + } } - } - }, - new ShardGroup() - { - ShardKeys = new List() + }, + new ShardGroup() { - new ShardKey() + ShardKeys = new List() { - Name = "ChainId", - Value = "tDVV", - Step = "", - StepType = StepType.None - }, - new ShardKey() - { - Name = "BlockHeight", - Value = "0", - Step = "10", - StepType = StepType.Floor + new ShardKey() + { + Name = "ChainId", + Value = "tDVV", + Step = "", + StepType = StepType.None + }, + new ShardKey() + { + Name = "BlockHeight", + Value = "0", + Step = "10", + StepType = StepType.Floor + } } } } }; - ShardInitSetting logEventIndexDto = new ShardInitSetting(); - logEventIndexDto.CollectionName = "LogEventIndex"; - logEventIndexDto.ShardGroups = new List() + var logEventIndexDto = new ShardInitSetting { - new ShardGroup() + CollectionName = "LogEventIndex", + ShardGroups = new List() { - ShardKeys = new List() + new ShardGroup() { - new ShardKey() - { - Name = "ChainId", - Value = "AELF", - Step = "", - StepType = StepType.None - }, - new ShardKey() + ShardKeys = new List() { - Name = "BlockHeight", - Value = "0", - Step = "2000", - StepType = StepType.Floor + new ShardKey() + { + Name = "ChainId", + Value = "AELF", + Step = "", + StepType = StepType.None + }, + new ShardKey() + { + Name = "BlockHeight", + Value = "0", + Step = "2000", + StepType = StepType.Floor + } } - } - }, - new ShardGroup() - { - ShardKeys = new List() + }, + new ShardGroup() { - new ShardKey() - { - Name = "ChainId", - Value = "tDVV", - Step = "", - StepType = StepType.None - }, - new ShardKey() + ShardKeys = new List() { - Name = "BlockHeight", - Value = "0", - Step = "1000", - StepType = StepType.Floor + new ShardKey() + { + Name = "ChainId", + Value = "tDVV", + Step = "", + StepType = StepType.None + }, + new ShardKey() + { + Name = "BlockHeight", + Value = "0", + Step = "1000", + StepType = StepType.Floor + } } } } From deaadc6d87779a3777e8870219597c24485fde01 Mon Sep 17 00:00:00 2001 From: stephen Date: Tue, 23 Apr 2024 15:20:52 +0800 Subject: [PATCH 07/20] feat: order by clause support sub object query --- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 46 ++-- .../Linq/QueryModelExtensions.cs | 71 +++-- .../Entities/BlockBase.cs | 28 +- .../Entities/BlockIndex.cs | 5 +- .../Entities/LogEvent.cs | 5 +- .../Entities/LogEventIndex.cs | 45 +-- .../ElasticsearchRepositoryTests.cs | 259 +++++++++++++----- 7 files changed, 304 insertions(+), 155 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index d4dc4c51..60eab9ae 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -25,8 +25,8 @@ public QueryAggregator GenerateElasticQuery(QueryModel queryModel) QueryAggregator = new QueryAggregator(); VisitQueryModel(queryModel); return QueryAggregator; - } - + } + public override void VisitQueryModel(QueryModel queryModel) { queryModel.SelectClause.Accept(this, queryModel); @@ -34,17 +34,17 @@ public override void VisitQueryModel(QueryModel queryModel) VisitBodyClauses(queryModel.BodyClauses, queryModel); VisitResultOperators(queryModel.ResultOperators, queryModel); } - + public override void VisitMainFromClause(MainFromClause fromClause, QueryModel queryModel) { if (fromClause.FromExpression is SubQueryExpression subQueryExpression) { VisitQueryModel(subQueryExpression.QueryModel); } - + base.VisitMainFromClause(fromClause, queryModel); } - + public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index) { var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser); @@ -66,9 +66,10 @@ public override void VisitWhereClause(WhereClause whereClause, QueryModel queryM QueryAggregator.Query = query; } + base.VisitWhereClause(whereClause, queryModel, index); } - + protected override void VisitResultOperators(ObservableCollection resultOperators, QueryModel queryModel) { @@ -78,7 +79,7 @@ protected override void VisitResultOperators(ObservableCollection>(); - + switch (groupResultOperator.KeySelector) { case MemberExpression memberExpression: @@ -99,20 +100,17 @@ protected override void VisitResultOperators(ObservableCollection new Tuple(memberExpression.Member.Name, memberExpression.Type))); break; } - - members.ForEach(property => - { - QueryAggregator.GroupByExpressions.Add(new GroupByProperties(property.Item1, property.Item2)); - }); + + members.ForEach(property => { QueryAggregator.GroupByExpressions.Add(new GroupByProperties(property.Item1, property.Item2)); }); } } - + base.VisitResultOperators(resultOperators, queryModel); } - - private static string GetCollectionNameKey(MemberExpression memberExpression) + + private string GetCollectionNameKey(MemberExpression memberExpression) { - string key = memberExpression.Member.Name; + var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); while (memberExpression.Expression != null) { memberExpression = memberExpression.Expression as MemberExpression; @@ -120,22 +118,26 @@ private static string GetCollectionNameKey(MemberExpression memberExpression) { break; } - key = memberExpression.Member.Name +"."+ key; + + key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name) + "." + key; return key; } + return key; } + + public override void VisitOrderByClause(OrderByClause orderByClause, QueryModel queryModel, int index) { foreach (var ordering in orderByClause.Orderings) { - var memberExpression = (MemberExpression) ordering.Expression; + var memberExpression = (MemberExpression)ordering.Expression; var direction = orderByClause.Orderings[0].OrderingDirection; - var propertyName = memberExpression.Member.Name; + var propertyName = GetCollectionNameKey(memberExpression); var type = memberExpression.Type; - QueryAggregator.OrderByExpressions.Add(new OrderProperties(propertyName, type, direction)); + QueryAggregator.OrderByExpressions.Add(new OrderProperties(propertyName, type, direction)); } - + base.VisitOrderByClause(orderByClause, queryModel, index); } } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs index 57c58f6f..728fa583 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryModelExtensions.cs @@ -17,36 +17,45 @@ public static List GetCollectionNameConditions(this Que private static void VisitQueryModel(List conditions, QueryModel queryModel) { var whereClauses = queryModel.BodyClauses.OfType().ToList(); - foreach (var predicate in whereClauses.Select(whereClause => (BinaryExpression)whereClause.Predicate)) + foreach (var predicate in whereClauses.Select(whereClause => whereClause.Predicate)) { - switch (predicate.Left) + switch (predicate) { - case BinaryExpression left: - VisitBinaryExpression(conditions, left); - break; - case SubQueryExpression leftSub: - VisitQueryModel(conditions, leftSub.QueryModel); - break; - } - - switch (predicate.Right) - { - case BinaryExpression right: - VisitBinaryExpression(conditions, right); + case BinaryExpression binaryExpression: + switch (binaryExpression.Left) + { + case BinaryExpression left: + VisitBinaryExpression(conditions, left); + break; + case SubQueryExpression leftSub: + VisitQueryModel(conditions, leftSub.QueryModel); + break; + } + + switch (binaryExpression.Right) + { + case BinaryExpression right: + VisitBinaryExpression(conditions, right); + break; + case SubQueryExpression rightSub: + VisitQueryModel(conditions, rightSub.QueryModel); + break; + } + + if (binaryExpression.Left is not BinaryExpression + && binaryExpression.Left is not SubQueryExpression + && binaryExpression.Right is not BinaryExpression + && binaryExpression.Right is not SubQueryExpression + && binaryExpression is BinaryExpression p) + { + VisitBinaryExpression(conditions, p); + } + break; - case SubQueryExpression rightSub: - VisitQueryModel(conditions, rightSub.QueryModel); + case SubQueryExpression subQueryExpression: + VisitQueryModel(conditions, subQueryExpression.QueryModel); break; } - - if (predicate.Left is not BinaryExpression - && predicate.Left is not SubQueryExpression - && predicate.Right is not BinaryExpression - && predicate.Right is not SubQueryExpression - && predicate is BinaryExpression p) - { - VisitBinaryExpression(conditions, p); - } } } @@ -58,15 +67,14 @@ private static void VisitBinaryExpression(List conditio var constantExpression = expression.Right as ConstantExpression; conditions.Add(new CollectionNameCondition { - - Key = GetCollectionNameKey(memberExpression), + Key = GetCollectionNameKey(memberExpression), Value = constantExpression.Value, Type = GetConditionType(expression.NodeType) }); - + return; } - + switch (expression.Left) { case SubQueryExpression leftSub: @@ -106,6 +114,7 @@ private static ConditionType GetConditionType(ExpressionType expressionType) throw new ArgumentOutOfRangeException(nameof(expressionType), expressionType, null); } } + private static string GetCollectionNameKey(MemberExpression memberExpression) { var key = memberExpression.Member.Name; @@ -116,9 +125,11 @@ private static string GetCollectionNameKey(MemberExpression memberExpression) { break; } - key = memberExpression.Member.Name +"."+ key; + + key = memberExpression.Member.Name + "." + key; return key; } + return key; } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs index 1b856208..6f36bb3e 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs @@ -3,22 +3,26 @@ namespace AElf.EntityMapping.Elasticsearch.Entities; -public class BlockBase: AElfIndexerEntity,IBlockchainData +public class BlockBase : AElfIndexerEntity, IBlockchainData { - [Keyword]public override string Id { get; set; } + [Keyword] public override string Id { get; set; } + [Keyword] - [ShardPropertyAttributes("ChainId",1)] + [ShardPropertyAttributes("ChainId", 1)] public string ChainId { get; set; } - [CollectionRouteKey] - [Keyword]public string BlockHash { get; set; } - [ShardPropertyAttributes("BlockHeight",3)] + + [CollectionRouteKey] [Keyword] public string BlockHash { get; set; } + + [ShardPropertyAttributes("BlockHeight", 3)] public long BlockHeight { get; set; } - [Keyword]public string PreviousBlockHash { get; set; } + + [Keyword] public string PreviousBlockHash { get; set; } public DateTime BlockTime { get; set; } - [Keyword]public string SignerPubkey { get; set; } - [Keyword]public string Signature { get; set; } - [ShardPropertyAttributes("Confirmed",2)] - public bool Confirmed{get;set;} - public Dictionary ExtraProperties {get;set;} + [Keyword] public string SignerPubkey { get; set; } + [Keyword] public string Signature { get; set; } + + [ShardPropertyAttributes("Confirmed", 2)] + public bool Confirmed { get; set; } + public Dictionary ExtraProperties { get; set; } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs index 5ede5cd7..02982609 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockIndex.cs @@ -3,15 +3,16 @@ namespace AElf.EntityMapping.Elasticsearch.Entities; -public class BlockIndex:BlockBase,IEntityMappingEntity +public class BlockIndex : BlockBase, IEntityMappingEntity { public List TransactionIds { get; set; } public int LogEventCount { get; set; } - [Keyword]public string TxnFee { get; set; } + [Keyword] public string TxnFee { get; set; } public FeeIndex Fee { get; set; } } public class FeeIndex { [Keyword] public string BlockFee { get; set; } + public long Fee { get; set; } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEvent.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEvent.cs index c4a2b763..7a503a64 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEvent.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEvent.cs @@ -3,12 +3,10 @@ namespace AElf.EntityMapping.Elasticsearch.Entities { - [NestedAttributes("LogEvents")] public class LogEvent : IBlockchainData { - [Keyword] - public string ChainId { get; set; } + [Keyword] public string ChainId { get; set; } [Keyword] public string BlockHash { get; set; } @@ -31,5 +29,4 @@ public class LogEvent : IBlockchainData public Dictionary ExtraProperties { get; set; } } - } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEventIndex.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEventIndex.cs index b5f4a2ac..038c6e0e 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEventIndex.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/LogEventIndex.cs @@ -4,40 +4,41 @@ namespace AElf.EntityMapping.Elasticsearch.Entities; -public class LogEventIndex:AElfIndexerEntity,IEntityMappingEntity,IBlockchainData +public class LogEventIndex : AElfIndexerEntity, IEntityMappingEntity, IBlockchainData { [Keyword] public override string Id { - get - { - return BlockHash + "_" + TransactionId + "_" + Index; - } + get { return BlockHash + "_" + TransactionId + "_" + Index; } } - [ShardPropertyAttributes("ChainId",1)] - [Keyword]public string ChainId { get; set; } - [Keyword]public string BlockHash { get; set; } - [Keyword]public string PreviousBlockHash { get; set; } + + [ShardPropertyAttributes("ChainId", 1)] + [Keyword] + public string ChainId { get; set; } + + [Keyword] public string BlockHash { get; set; } + [Keyword] public string PreviousBlockHash { get; set; } + /// /// block height /// - [ShardPropertyAttributes("BlockHeight",2)] + [ShardPropertyAttributes("BlockHeight", 2)] public long BlockHeight { get; set; } - - [Keyword]public string TransactionId { get; set; } - + + [Keyword] public string TransactionId { get; set; } + public DateTime BlockTime { get; set; } - - [Keyword]public string ContractAddress { get; set; } - - [Keyword]public string EventName { get; set; } - + + [Keyword] public string ContractAddress { get; set; } + + [Keyword] public string EventName { get; set; } + /// /// The ranking position of the event within the transaction /// public int Index { get; set; } - - public bool Confirmed{get;set;} - - public Dictionary ExtraProperties {get;set;} + + public bool Confirmed { get; set; } + + public Dictionary ExtraProperties { get; set; } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index 3adf29e5..dca3c924 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -1,7 +1,10 @@ +using System.Linq.Dynamic.Core; using System.Linq.Expressions; +using System.Transactions; using AElf.EntityMapping.Elasticsearch.Entities; using AElf.EntityMapping.Elasticsearch.Services; using AElf.EntityMapping.Options; +using AElf.EntityMapping.Repositories; using Microsoft.Extensions.Options; using Shouldly; using Xunit; @@ -11,12 +14,14 @@ namespace AElf.EntityMapping.Elasticsearch.Repositories; public class ElasticsearchRepositoryTests : AElfElasticsearchTestBase { private readonly IElasticsearchRepository _elasticsearchRepository; + private readonly IElasticsearchRepository _transactionIndexRepository; private readonly IElasticIndexService _elasticIndexService; private readonly AElfEntityMappingOptions _option; public ElasticsearchRepositoryTests() { _elasticsearchRepository = GetRequiredService>(); + _transactionIndexRepository = GetRequiredService>(); _elasticIndexService = GetRequiredService(); _option = GetRequiredService>().Value; } @@ -24,7 +29,7 @@ public ElasticsearchRepositoryTests() [Fact] public async Task AddAsync() { - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -34,7 +39,7 @@ public async Task AddAsync() ChainId = "AELF" }; await _elasticsearchRepository.AddAsync(blockIndex); - + var queryable = await _elasticsearchRepository.GetQueryableAsync(); Expression> expression = p => p.ChainId == blockIndex.ChainId && p.BlockHeight == blockIndex.BlockHeight; @@ -43,13 +48,12 @@ public async Task AddAsync() Assert.True(results.First().Id == blockIndex.Id); Assert.True(results.First().BlockHeight == blockIndex.BlockHeight); await _elasticsearchRepository.DeleteAsync(blockIndex); - } - + [Fact] public async Task AddOrUpdateAsyncTest() { - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -59,7 +63,7 @@ public async Task AddOrUpdateAsyncTest() ChainId = "AELF" }; await _elasticsearchRepository.AddOrUpdateAsync(blockIndex); - + var queryable = await _elasticsearchRepository.GetQueryableAsync(); Expression> expression = p => p.ChainId == blockIndex.ChainId && p.BlockHeight == blockIndex.BlockHeight && p.Id == blockIndex.Id; @@ -68,7 +72,7 @@ public async Task AddOrUpdateAsyncTest() Assert.True(results.First().Id == blockIndex.Id); Assert.True(results.First().BlockHeight == blockIndex.BlockHeight); - var blockIndex2 = new BlockIndex + var blockIndex2 = new BlockIndex { Id = "block002", BlockHash = "BlockHash001", @@ -78,7 +82,7 @@ public async Task AddOrUpdateAsyncTest() ChainId = "AELF" }; await _elasticsearchRepository.AddOrUpdateAsync(blockIndex2); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex.ChainId && p.BlockHeight == blockIndex.BlockHeight && p.Id == blockIndex2.Id; @@ -86,7 +90,7 @@ public async Task AddOrUpdateAsyncTest() Assert.True(!results.IsNullOrEmpty()); Assert.True(results.First().Id == blockIndex2.Id); Assert.True(results.First().BlockHeight == blockIndex2.BlockHeight); - + expression = p => p.ChainId == blockIndex.ChainId && p.BlockHeight == blockIndex.BlockHeight; results = queryable.Where(expression).ToList(); @@ -95,11 +99,11 @@ public async Task AddOrUpdateAsyncTest() await _elasticsearchRepository.DeleteAsync(blockIndex); await _elasticsearchRepository.DeleteAsync(blockIndex2); } - + [Fact] public async Task AddOrUpdateManyAsyncTest() { - var blockIndex1 = new BlockIndex + var blockIndex1 = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -108,7 +112,7 @@ public async Task AddOrUpdateManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var blockIndex2 = new BlockIndex + var blockIndex2 = new BlockIndex { Id = "block002", BlockHash = "BlockHash002", @@ -117,7 +121,7 @@ public async Task AddOrUpdateManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var blockIndex3 = new BlockIndex + var blockIndex3 = new BlockIndex { Id = "block003", BlockHash = "BlockHash003", @@ -126,23 +130,23 @@ public async Task AddOrUpdateManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var bulkList = new List {blockIndex1, blockIndex2, blockIndex3}; + var bulkList = new List { blockIndex1, blockIndex2, blockIndex3 }; await _elasticsearchRepository.AddOrUpdateManyAsync(bulkList); - + var queryable = await _elasticsearchRepository.GetQueryableAsync(); Expression> expression = p => p.ChainId == blockIndex1.ChainId && p.BlockHeight == blockIndex1.BlockHeight && p.Id == blockIndex1.Id; var results = queryable.Where(expression).ToList(); Assert.True(!results.IsNullOrEmpty()); Assert.True(results.First().Id == blockIndex1.Id); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex2.ChainId && p.BlockHeight == blockIndex2.BlockHeight && p.Id == blockIndex2.Id; results = queryable.Where(expression).ToList(); Assert.True(!results.IsNullOrEmpty()); Assert.True(results.First().Id == blockIndex2.Id); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex3.ChainId && p.BlockHeight == blockIndex3.BlockHeight && p.Id == blockIndex3.Id; @@ -153,11 +157,11 @@ public async Task AddOrUpdateManyAsyncTest() await _elasticsearchRepository.DeleteAsync(blockIndex2); await _elasticsearchRepository.DeleteAsync(blockIndex3); } - + [Fact] public async Task UpdateAsyncTest() { - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -169,7 +173,7 @@ public async Task UpdateAsyncTest() await _elasticsearchRepository.AddAsync(blockIndex); blockIndex.LogEventCount = 20; await _elasticsearchRepository.UpdateAsync(blockIndex); - + var queryable = await _elasticsearchRepository.GetQueryableAsync(); Expression> expression = p => p.ChainId == blockIndex.ChainId && p.Id == blockIndex.Id; @@ -183,7 +187,7 @@ public async Task UpdateAsyncTest() [Fact] public async Task UpdateManyAsyncTest() { - var blockIndex1 = new BlockIndex + var blockIndex1 = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -192,7 +196,7 @@ public async Task UpdateManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var blockIndex2 = new BlockIndex + var blockIndex2 = new BlockIndex { Id = "block002", BlockHash = "BlockHash002", @@ -201,7 +205,7 @@ public async Task UpdateManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var blockIndex3 = new BlockIndex + var blockIndex3 = new BlockIndex { Id = "block003", BlockHash = "BlockHash003", @@ -210,9 +214,9 @@ public async Task UpdateManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var bulkList = new List {blockIndex1, blockIndex2, blockIndex3}; + var bulkList = new List { blockIndex1, blockIndex2, blockIndex3 }; await _elasticsearchRepository.AddOrUpdateManyAsync(bulkList); - + var queryable = await _elasticsearchRepository.GetQueryableAsync(); Expression> expression = p => p.ChainId == blockIndex1.ChainId && p.BlockHeight == blockIndex1.BlockHeight && p.Id == blockIndex1.Id; @@ -220,7 +224,7 @@ public async Task UpdateManyAsyncTest() results.ShouldNotBeNull(); results.First().BlockHash.ShouldBe(blockIndex1.BlockHash); results.First().LogEventCount.ShouldBe(blockIndex1.LogEventCount); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex2.ChainId && p.BlockHeight == blockIndex2.BlockHeight && p.Id == blockIndex2.Id; @@ -228,7 +232,7 @@ public async Task UpdateManyAsyncTest() results.ShouldNotBeNull(); results.First().BlockHash.ShouldBe(blockIndex2.BlockHash); results.First().LogEventCount.ShouldBe(blockIndex2.LogEventCount); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex3.ChainId && p.BlockHeight == blockIndex3.BlockHeight && p.Id == blockIndex3.Id; @@ -236,27 +240,27 @@ public async Task UpdateManyAsyncTest() results.ShouldNotBeNull(); results.First().BlockHash.ShouldBe(blockIndex3.BlockHash); results.First().LogEventCount.ShouldBe(blockIndex3.LogEventCount); - + blockIndex1.LogEventCount = 100; blockIndex2.LogEventCount = 200; blockIndex3.LogEventCount = 300; - var bulkUpdateList = new List {blockIndex1, blockIndex2, blockIndex3}; + var bulkUpdateList = new List { blockIndex1, blockIndex2, blockIndex3 }; await _elasticsearchRepository.UpdateManyAsync(bulkUpdateList); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex1.ChainId && p.BlockHeight == blockIndex1.BlockHeight && p.Id == blockIndex1.Id; results = queryable.Where(expression).ToList(); results.ShouldNotBeNull(); results.First().LogEventCount.ShouldBe(100); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex2.ChainId && p.BlockHeight == blockIndex2.BlockHeight && p.Id == blockIndex2.Id; results = queryable.Where(expression).ToList(); results.ShouldNotBeNull(); results.First().LogEventCount.ShouldBe(200); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex3.ChainId && p.BlockHeight == blockIndex3.BlockHeight && p.Id == blockIndex3.Id; @@ -264,11 +268,11 @@ public async Task UpdateManyAsyncTest() results.ShouldNotBeNull(); results.First().LogEventCount.ShouldBe(300); } - + [Fact] public async Task DeleteAsyncByEntityTest() { - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -283,9 +287,9 @@ public async Task DeleteAsyncByEntityTest() p.ChainId == blockIndex.ChainId && p.Id == blockIndex.Id; var results = queryable.Where(expression).ToList(); Assert.True(!results.IsNullOrEmpty()); - + await _elasticsearchRepository.DeleteAsync(blockIndex); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex.ChainId && p.Id == blockIndex.Id; @@ -296,7 +300,7 @@ public async Task DeleteAsyncByEntityTest() [Fact] public async Task DeleteAsyncByIdTest() { - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -313,7 +317,7 @@ public async Task DeleteAsyncByIdTest() p.ChainId == blockIndex.ChainId && p.Id == blockIndex.Id; var results = queryable.Where(expression).ToList(); Assert.True(!results.IsNullOrEmpty()); - + await _elasticsearchRepository.DeleteAsync(blockIndex.Id); Thread.Sleep(1000); queryable = await _elasticsearchRepository.GetQueryableAsync(); @@ -322,11 +326,11 @@ public async Task DeleteAsyncByIdTest() results = queryable.Where(expression).ToList(); Assert.True(results.IsNullOrEmpty()); } - + [Fact] public async Task DeleteManyAsyncTest() { - var blockIndex1 = new BlockIndex + var blockIndex1 = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -335,7 +339,7 @@ public async Task DeleteManyAsyncTest() LogEventCount = 10, ChainId = "AELF" }; - var blockIndex2 = new BlockIndex + var blockIndex2 = new BlockIndex { Id = "block002", BlockHash = "BlockHash002", @@ -344,7 +348,7 @@ public async Task DeleteManyAsyncTest() LogEventCount = 20, ChainId = "AELF" }; - var blockIndex3 = new BlockIndex + var blockIndex3 = new BlockIndex { Id = "block003", BlockHash = "BlockHash003", @@ -353,7 +357,7 @@ public async Task DeleteManyAsyncTest() LogEventCount = 30, ChainId = "AELF" }; - var blockIndex30 = new BlockIndex + var blockIndex30 = new BlockIndex { Id = "block030", BlockHash = "BlockHash030", @@ -362,35 +366,35 @@ public async Task DeleteManyAsyncTest() LogEventCount = 30, ChainId = "AELF" }; - var bulkList = new List {blockIndex1, blockIndex2, blockIndex3,blockIndex30}; + var bulkList = new List { blockIndex1, blockIndex2, blockIndex3, blockIndex30 }; await _elasticsearchRepository.AddOrUpdateManyAsync(bulkList); await _elasticsearchRepository.DeleteManyAsync(bulkList); var queryable = await _elasticsearchRepository.GetQueryableAsync(); - + Expression> expression = p => p.ChainId == blockIndex1.ChainId && p.Id == blockIndex1.Id; Thread.Sleep(500); var results = queryable.Where(expression).ToList(); Assert.True(results.IsNullOrEmpty()); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex2.ChainId && p.Id == blockIndex2.Id; results = queryable.Where(expression).ToList(); Assert.True(results.IsNullOrEmpty()); - + queryable = await _elasticsearchRepository.GetQueryableAsync(); expression = p => p.ChainId == blockIndex30.ChainId && p.Id == blockIndex30.Id; results = queryable.Where(expression).ToList(); Assert.True(results.IsNullOrEmpty()); } - + [Fact] public async Task Get_Test() { - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -408,7 +412,7 @@ public async Task Get_Test() block.BlockHeight.ShouldBe(blockIndex.BlockHeight); block.LogEventCount.ShouldBe(blockIndex.LogEventCount); block.ChainId.ShouldBe(blockIndex.ChainId); - + for (int i = 1; i <= 7; i++) { await _elasticsearchRepository.AddAsync(new BlockIndex @@ -421,7 +425,7 @@ await _elasticsearchRepository.AddAsync(new BlockIndex ChainId = "AELF" }); } - + block = await _elasticsearchRepository.GetAsync("block7"); block.Id.ShouldBe("block7"); } @@ -430,7 +434,7 @@ await _elasticsearchRepository.AddAsync(new BlockIndex public async Task Get_SpecificIndex_Test() { var indexName = $"{_option.CollectionPrefix}.block".ToLower(); - var blockIndex = new BlockIndex + var blockIndex = new BlockIndex { Id = "block001", BlockHash = "BlockHash001", @@ -507,7 +511,7 @@ public async Task GetList_SpecificIndex_Test() { var indexName = $"{_option.CollectionPrefix}.block".ToLower(); await _elasticIndexService.CreateIndexAsync(indexName, typeof(BlockIndex), 1, 0); - + for (int i = 1; i <= 7; i++) { var blockIndex = new BlockIndex @@ -551,6 +555,44 @@ public async Task GetList_SpecificIndex_Test() list[0].BlockHeight.ShouldBe(6); } + private async Task ClearTransactionIndex(string chainId, long startBlockNumber, long endBlockNumber) + { + Expression> expression = p => p.ChainId == chainId && p.BlockHeight >= startBlockNumber && p.BlockHeight <= endBlockNumber; + var queryable = await _transactionIndexRepository.GetQueryableAsync(); + var filterList = queryable.Where(expression).ToList(); + foreach (var deleteTransaction in filterList) + { + await _transactionIndexRepository.DeleteAsync(deleteTransaction); + } + } + + [Fact] + public async Task GetList_Nested_Test() + { + //clear data for unit test + ClearTransactionIndex("AELF", 100, 110); + + Thread.Sleep(2000); + //Unit Test 14 + var transaction_100_1 = MockNewTransactionEtoData(100, false, "token_contract_address", "DonateResourceToken"); + var transaction_100_2 = MockNewTransactionEtoData(100, false, "", ""); + var transaction_100_3 = MockNewTransactionEtoData(100, false, "consensus_contract_address", "UpdateValue"); + var transaction_110 = MockNewTransactionEtoData(110, true, "consensus_contract_address", "UpdateTinyBlockInformation"); + await _transactionIndexRepository.AddAsync(transaction_100_1); + await _transactionIndexRepository.AddAsync(transaction_100_2); + await _transactionIndexRepository.AddAsync(transaction_100_3); + await _transactionIndexRepository.AddAsync(transaction_110); + Thread.Sleep(2000); + + var chainId = "AELF"; + Expression> mustQuery = p => p.LogEvents.Any(i => i.ChainId == chainId && i.BlockHeight >= 100 && i.BlockHeight <= 110); + mustQuery = p => p.LogEvents.Any(i => i.ChainId == "AELF" && i.TransactionId == transaction_100_1.TransactionId); + var queryable = await _transactionIndexRepository.GetQueryableAsync(); + var filterList = queryable.Where(mustQuery).ToList(); + filterList.Count.ShouldBe(1); + } + + [Fact] public async Task GetList_MultipleChain_Test() { @@ -568,11 +610,12 @@ public async Task GetList_MultipleChain_Test() Fee = new FeeIndex() { BlockFee = "BlockHash" + i, + Fee = i % 4 } }; await _elasticsearchRepository.AddAsync(blockIndex); } - + for (int i = 1; i <= 11; i++) { var blockIndex = new BlockIndex @@ -586,15 +629,105 @@ public async Task GetList_MultipleChain_Test() }; await _elasticsearchRepository.AddAsync(blockIndex); } - - /*var list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="AELF" && o.BlockHeight >= 0); - list.Count.ShouldBe(7); - - list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="tDVV" && o.BlockHeight >= 0); - list.Count.ShouldBe(11);*/ - - var list = await _elasticsearchRepository.GetListAsync(o =>o.ChainId =="AELF" && o.BlockHeight >= 0 && o.Fee.BlockFee == "BlockHash2" ); + + var chainId = "AELF"; + var list = await _elasticsearchRepository.GetListAsync(o => o.ChainId == chainId && o.BlockHeight >= 0 && o.Fee.BlockFee == "BlockHash2"); list.Count.ShouldBe(1); + foreach (var e in list) + { + e.ChainId.ShouldBe(chainId); + e.BlockHeight.ShouldBeGreaterThan(0); + e.Fee.BlockFee.ShouldBe("BlockHash2"); + } + + var queryable = await _elasticsearchRepository.GetQueryableAsync(); + var list1 = queryable.Where(o => o.ChainId == chainId).OrderByDescending(o => o.LogEventCount).ToList(); + list1.Count.ShouldBe(7); + list1.First().LogEventCount.ShouldBe(7); + foreach (var e in list1) + { + e.ChainId.ShouldBe(chainId); + } + + var list2 = queryable.Where(o => o.ChainId == chainId && o.BlockHeight >= 0 && o.Fee.Fee == 3).ToList(); + list2.Count.ShouldBe(2); + foreach (var e in list2) + { + e.ChainId.ShouldBe(chainId); + e.BlockHeight.ShouldBeGreaterThanOrEqualTo(0); + e.Fee.Fee.ShouldBe(3); + } + + var list3 = queryable.Where(o => o.ChainId == chainId && o.BlockHeight >= 2).OrderBy(o => o.Fee.Fee).ToList(); + list3.Count.ShouldBe(6); + list3.First().Fee.Fee.ShouldBe(0); + foreach (var e in list3) + { + e.ChainId.ShouldBe(chainId); + e.BlockHeight.ShouldBeGreaterThanOrEqualTo(2); + } + } + + + public static TransactionIndex MockNewTransactionEtoData(long blockHeight, bool isConfirmed, string contractAddress, string eventName) + { + string currentBlockHash = CreateBlockHash(); + string transactionId = CreateBlockHash(); + var newTransaction = new TransactionIndex() + { + Id = transactionId, + TransactionId = transactionId, + ChainId = "AELF", + From = "2pL7foxBhMC1RVZMUEtkvYK4pWWaiLHBAQcXFdzfD5oZjYSr3e", + To = "pGa4e5hNGsgkfjEGm72TEvbF7aRDqKBd4LuXtab4ucMbXLcgJ", + BlockHash = currentBlockHash, + BlockHeight = blockHeight, + BlockTime = DateTime.Now, + MethodName = "UpdateValue", + Params = + "CiIKINnB4HhmTpMScNl9T4hNoR1w8dOpx0O684p+pwfm6uOkEiIKINZUk1v+szUqMZZ3w0phLn7qqOX+h+uD1fdP59LYm9CsIiIKIPGQ8ga2zO3CXXtZkjlMma6CI7VSQFA7ZMYrIU6zGT/uKgYInMjcmAYwAVDPAlqpAQqCATA0YmNkMWM4ODdjZDBlZGJkNGNjZjhkOWQyYjNmNzJlNzI1MTFhYTYxODMxOTk2MDAzMTM2ODdiYTZjNTgzZjEzYzNkNmQ3MTZmYTQwZGY4NjA0YWFlZDBmY2FiMzExMzVmZTNjMmQ0NWMwMDk4MDBjMDc1MjU0YTM3ODJiNGM0ZGISIgog8ZDyBrbM7cJde1mSOUyZroIjtVJAUDtkxishTrMZP+5g0AI=", + Signature = + "1KblGpvuuo+HSDdh0OhRq/vg3Ts4HoqcIwBeni/356pdEbgnnR2yqbpgvzNs+oNeBb4Ux2kE1XY9lk+p60LfWgA=", + Index = 0, + Status = TransactionStatus.Committed, + Confirmed = isConfirmed, + ExtraProperties = new Dictionary() + { + ["Version"] = "0", + ["RefBlockNumber"] = "335", + ["RefBlockPrefix"] = "156ff372", + ["Bloom"] = + "AAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAEAAAAAAAAgAAABAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAgAAAAAAAAAABAAAAAAAAAAAAAAIQAAAAABAAAAAgAAAAAAAAAAAAAAAAABACAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIgAAAAAAAAAAAAAAAAAAAAAAAEAABAAAAAAAAAAAAAAAAAAAAgAAAAgAAAAABAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAgAAAAAAAAAAAAAA==", + ["ReturnValue"] = "", + ["Error"] = "" + }, + LogEvents = new List() + { + new LogEvent() + { + ChainId = "AELF", + BlockHash = currentBlockHash, + BlockHeight = blockHeight, + BlockTime = DateTime.Now, + Confirmed = isConfirmed, + TransactionId = transactionId, + ContractAddress = contractAddress, + EventName = eventName, + Index = 0, + ExtraProperties = new Dictionary() + { + ["Indexed"] = + "[ \"CoIBMDRiY2QxYzg4N2NkMGVkYmQ0Y2NmOGQ5ZDJiM2Y3MmU3MjUxMWFhNjE4MzE5OTYwMDMxMzY4N2JhNmM1ODNmMTNjM2Q2ZDcxNmZhNDBkZjg2MDRhYWVkMGZjYWIzMTEzNWZlM2MyZDQ1YzAwOTgwMGMwNzUyNTRhMzc4MmI0YzRkYg==\", \"EgsIxNqlmQYQ4M74LQ==\", \"GhpVcGRhdGVUaW55QmxvY2tJbmZvcm1hdGlvbg==\", \"ILAi\", \"KiIKID3kBhYftHeFZBYS6VOXPeigGAAwZWM85SlzN48xJARW\" ]", + ["NonIndexed"] = "" + } + } + } + }; + return newTransaction; + } + + public static string CreateBlockHash() + { + return Guid.NewGuid().ToString("N") + Guid.NewGuid().ToString("N"); } - } \ No newline at end of file From ef49dc9a36fb557b6f60bf7428198ba9525314d7 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 25 Apr 2024 15:03:47 +0800 Subject: [PATCH 08/20] feat: add wildcard query unit test --- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 8 +-- .../Linq/ElasticsearchQueryExecutor.cs | 2 +- .../Sharding/ShardingKeyProvider.cs | 4 +- .../ElasticsearchRepositoryTests.cs | 63 ++++++++++++++++++- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index 60eab9ae..f8a1c70a 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -92,12 +92,12 @@ protected override void VisitResultOperators(ObservableCollection(memberExpression.Member.Name, memberExpression.Type)); + members.Add(new Tuple(GetFullNameKey(memberExpression), memberExpression.Type)); break; case NewExpression newExpression: members.AddRange(newExpression.Arguments .Cast() - .Select(memberExpression => new Tuple(memberExpression.Member.Name, memberExpression.Type))); + .Select(memberExpression => new Tuple(GetFullNameKey(memberExpression), memberExpression.Type))); break; } @@ -108,7 +108,7 @@ protected override void VisitResultOperators(ObservableCollection ExecuteCollection(QueryModel queryModel) { descriptor.MatchAll(); } - + if (queryAggregator.OrderByExpressions.Any()) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/ShardingKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/ShardingKeyProvider.cs index f9697e58..243d8ffe 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/ShardingKeyProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Sharding/ShardingKeyProvider.cs @@ -70,8 +70,8 @@ public async Task> GetCollectionNameAsync(List(); } - List> shardingKeyInfos = GetShardKeyInfoList(); - List> filterShardingKeyInfos = shardingKeyInfos; + var shardingKeyInfos = GetShardKeyInfoList(); + var filterShardingKeyInfos = shardingKeyInfos; List filterConditions = new List(); foreach (var condition in conditions) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index dca3c924..cc26451d 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -1,10 +1,8 @@ -using System.Linq.Dynamic.Core; using System.Linq.Expressions; using System.Transactions; using AElf.EntityMapping.Elasticsearch.Entities; using AElf.EntityMapping.Elasticsearch.Services; using AElf.EntityMapping.Options; -using AElf.EntityMapping.Repositories; using Microsoft.Extensions.Options; using Shouldly; using Xunit; @@ -668,6 +666,67 @@ public async Task GetList_MultipleChain_Test() } } + [Fact] + public async Task GetList_WildCard_Test() + { + for (int i = 1; i <= 7; i++) + { + var blockIndex = new BlockIndex + { + Id = "block" + i, + BlockHash = "BlockHash" + i, + BlockHeight = i, + BlockTime = DateTime.Now.AddDays(-10 + i), + LogEventCount = i, + ChainId = "AELF", + TxnFee = "BlockHash" + i, + Fee = new FeeIndex() + { + BlockFee = "BlockHash" + i, + Fee = i % 4 + } + }; + await _elasticsearchRepository.AddAsync(blockIndex); + } + + for (int i = 1; i <= 11; i++) + { + var blockIndex = new BlockIndex + { + Id = "block" + i, + BlockHash = "BlockHash" + i, + BlockHeight = i, + BlockTime = DateTime.Now.AddDays(-10 + i), + LogEventCount = i, + ChainId = "tDVV" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + } + + var queryable = await _elasticsearchRepository.GetQueryableAsync(); + var list4 = queryable.Where(o => o.ChainId == "AELF" && o.BlockHash.StartsWith("BlockHash")).ToList(); + list4.Count.ShouldBe(7); + + var list5 = queryable.Where(o => o.ChainId == "AELF" && o.Id.Contains("6")).ToList(); + list5.Count.ShouldBe(1); + list5.First().Id.ShouldBe("block6"); + + var list6 = queryable.Where(o => o.ChainId == "AELF" && o.BlockHash.EndsWith("7")).ToList(); + list6.Count.ShouldBe(1); + list6.First().Id.ShouldBe("block7"); + + var list7 = queryable.Where(o => o.ChainId == "AELF" && o.Fee.BlockFee.StartsWith("BlockHash")).ToList(); + list7.Count.ShouldBe(7); + + var list8 = queryable.Where(o => o.ChainId == "AELF" && o.Fee.BlockFee.Contains("6")).ToList(); + list8.Count.ShouldBe(1); + list8.First().Id.ShouldBe("block6"); + + var list9 = queryable.Where(o => o.ChainId == "AELF" && o.Fee.BlockFee.EndsWith("7")).ToList(); + list9.Count.ShouldBe(1); + list9.First().Id.ShouldBe("block7"); + } + public static TransactionIndex MockNewTransactionEtoData(long blockHeight, bool isConfirmed, string contractAddress, string eventName) { From 403cca340574f2f4b75788da56ca39d94ad7b168 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 25 Apr 2024 18:36:08 +0800 Subject: [PATCH 09/20] feat: del duplicated code --- .../Linq/GeneratorExpressionTreeVisitor.cs | 2 -- .../Sharding/CollectionRouteKeyProvider.cs | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 9811921d..01771c0f 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -399,10 +399,8 @@ private Node HandleStringProperty(Expression expression) { case ExpressionType.Equal: return new TermNode(PropertyName, Value); - // return new MatchPhraseNode(PropertyName, Value); case ExpressionType.NotEqual: return new NotNode(new TermNode(PropertyName, Value)); - //return new NotNode(new MatchPhraseNode(PropertyName, Value)); default: return null; } diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs index 992cdd92..6615a9ad 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs @@ -119,10 +119,6 @@ public async Task> GetCollectionNameAsync(List(s => - // s.Index(collectionRouteKeyIndexName).Size(10000).Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) - // .Collapse(c => c.Field(f=>f.CollectionName)).Aggregations(a => a - // .Cardinality("courseAgg", ca => ca.Field(f=>f.CollectionName)))); var result = await client.SearchAsync(s => s.Index(collectionRouteKeyIndexName) .Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) From cb15560d77c81968d77b436d307ea36b5909a327 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 25 Apr 2024 18:37:23 +0800 Subject: [PATCH 10/20] feat: del duplicate code --- .../Linq/ElasticsearchQueryExecutor.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index e153e379..9ee8fbf0 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -238,7 +238,6 @@ public T ExecuteScalar(QueryModel queryModel) { descriptor.Query(q => queryAggregator.Query); } - var dsl = _elasticClient.RequestResponseSerializer.SerializeToString(descriptor); return descriptor; }); if (!response.IsValid) From 08fbbcf6ba35bf43ab3de7d82167e60904f3396e Mon Sep 17 00:00:00 2001 From: stephen Date: Sun, 28 Apr 2024 16:42:22 +0800 Subject: [PATCH 11/20] feat: optimize get index method --- .../ElasticsearchCollectionNameProvider.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs index e4a6dc42..2d45ff20 100644 --- a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs @@ -38,19 +38,17 @@ protected override async Task> GetCollectionNameAsync(List { GetDefaultCollectionName() }; - - var shardKeyCollectionNames = await _shardingKeyProvider.GetCollectionNameAsync(conditions); - var routeKeyCollectionNames = - await _collectionRouteKeyProvider.GetCollectionNameAsync(conditions); - if (shardKeyCollectionNames.Count > 0 && routeKeyCollectionNames.Count > 0) + var shardKeyCollectionNames = await _shardingKeyProvider.GetCollectionNameAsync(conditions); + if (shardKeyCollectionNames.IsNullOrEmpty()) { - return shardKeyCollectionNames.Intersect(routeKeyCollectionNames).ToList(); + return await _collectionRouteKeyProvider.GetCollectionNameAsync(conditions); } - return shardKeyCollectionNames.Concat(routeKeyCollectionNames).ToList(); + + return shardKeyCollectionNames; } - protected override async Task> GetCollectionNameByEntityAsync(TEntity entity) + protected override async Task> GetCollectionNameByEntityAsync(TEntity entity) { if (entity == null) return new List { GetDefaultCollectionName() }; @@ -65,7 +63,7 @@ protected override async Task> GetCollectionNameByEntityAsync(List< { if (entities == null || entities.Count == 0) return new List { GetDefaultCollectionName() }; - + return _shardingKeyProvider.IsShardingCollection() ? await _shardingKeyProvider.GetCollectionNameAsync(entities) : new List { GetDefaultCollectionName() }; @@ -73,7 +71,7 @@ protected override async Task> GetCollectionNameByEntityAsync(List< protected override async Task GetCollectionNameByIdAsync(TKey id) { - if (!_shardingKeyProvider.IsShardingCollection()) + if (!_shardingKeyProvider.IsShardingCollection()) return GetDefaultCollectionName(); return await _collectionRouteKeyProvider.GetCollectionNameAsync(id.ToString()); } From 2dbd2e6fd122c53c10ef69aff076d90a8c7ee15e Mon Sep 17 00:00:00 2001 From: zhxymh Date: Wed, 8 May 2024 11:54:20 +0800 Subject: [PATCH 12/20] feat: add settings for create index --- .../Services/ElasticIndexService.cs | 17 ++++++++++++++--- .../Services/IElasticIndexService.cs | 3 ++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs index 2aeb01d3..454c7720 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs @@ -38,7 +38,7 @@ public Task GetElasticsearchClientAsync() return Task.FromResult(_elasticsearchClientProvider.GetClient()); } - public async Task CreateIndexAsync(string indexName, Type type, int shard = 1, int numberOfReplicas = 1) + public async Task CreateIndexAsync(string indexName, Type type, int shard = 1, int numberOfReplicas = 1, Dictionary indexSettings = null) { if (!type.IsClass || type.IsAbstract || !typeof(IEntityMappingEntity).IsAssignableFrom(type)) { @@ -60,8 +60,19 @@ public async Task CreateIndexAsync(string indexName, Type type, int shard = 1, i ss => ss.Index(indexName) .Settings( - o => o.NumberOfShards(shard).NumberOfReplicas(numberOfReplicas) - .Setting("max_result_window", _elasticsearchOptions.MaxResultWindow)) + o => + { + var setting = o.NumberOfShards(shard).NumberOfReplicas(numberOfReplicas) + .Setting("max_result_window", _elasticsearchOptions.MaxResultWindow); + if (indexSettings != null) + { + foreach (var indexSetting in indexSettings) + { + setting.Setting(indexSetting.Key, indexSetting.Value); + } + } + return setting; + }) .Map(m => m.AutoMap(type))); if (!result.Acknowledged) throw new ElasticsearchException($"Create Index {indexName} failed : " + diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs index 6162f6dd..6136c3f6 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs @@ -2,7 +2,8 @@ namespace AElf.EntityMapping.Elasticsearch.Services; public interface IElasticIndexService { - Task CreateIndexAsync(string indexName, Type indexEntityType, int shard = 1, int numberOfReplicas = 1); + Task CreateIndexAsync(string indexName, Type indexEntityType, int shard = 1, int numberOfReplicas = 1, + Dictionary indexSettings = null); Task CreateIndexTemplateAsync(string indexTemplateName, string indexName, Type indexEntityType, int numberOfShards, int numberOfReplicas); From c5a50dcdede01c2f0e9d0b7efb19b86ab04c8759 Mon Sep 17 00:00:00 2001 From: zhxymh Date: Mon, 13 May 2024 17:20:45 +0800 Subject: [PATCH 13/20] feat: support search after --- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 52 ++++++++------- .../Linq/ElasticsearchQueryExecutor.cs | 10 ++- .../Linq/ElasticsearchQueryable.cs | 5 +- .../Linq/QueryAggregator.cs | 1 + .../AElf.EntityMapping.csproj | 2 + .../ElasticsearchRepositoryTests.cs | 64 +++++++++++++++++++ 6 files changed, 105 insertions(+), 29 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index f8a1c70a..5c664e71 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -1,5 +1,6 @@ using System.Collections.ObjectModel; using System.Linq.Expressions; +using AElf.EntityMapping.Linq; using Nest; using Remotion.Linq; using Remotion.Linq.Clauses; @@ -75,33 +76,36 @@ protected override void VisitResultOperators(ObservableCollection>(); - - switch (groupResultOperator.KeySelector) + case SkipResultOperator skipResultOperator: + QueryAggregator.Skip = skipResultOperator.GetConstantCount(); + break; + case TakeResultOperator takeResultOperator: + QueryAggregator.Take = takeResultOperator.GetConstantCount(); + break; + case GroupResultOperator groupResultOperator: { - case MemberExpression memberExpression: - members.Add(new Tuple(GetFullNameKey(memberExpression), memberExpression.Type)); - break; - case NewExpression newExpression: - members.AddRange(newExpression.Arguments - .Cast() - .Select(memberExpression => new Tuple(GetFullNameKey(memberExpression), memberExpression.Type))); - break; + var members = new List>(); + + switch (groupResultOperator.KeySelector) + { + case MemberExpression memberExpression: + members.Add(new Tuple(GetFullNameKey(memberExpression), memberExpression.Type)); + break; + case NewExpression newExpression: + members.AddRange(newExpression.Arguments + .Cast() + .Select(memberExpression => new Tuple(GetFullNameKey(memberExpression), memberExpression.Type))); + break; + } + + members.ForEach(property => { QueryAggregator.GroupByExpressions.Add(new GroupByProperties(property.Item1, property.Item2)); }); + break; } - - members.ForEach(property => { QueryAggregator.GroupByExpressions.Add(new GroupByProperties(property.Item1, property.Item2)); }); + case AfterResultOperator afterResultOperator: + QueryAggregator.After = afterResultOperator.GetConstantPosition(); + break; } } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index 9ee8fbf0..8e93a67e 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -86,7 +86,12 @@ public IEnumerable ExecuteCollection(QueryModel queryModel) descriptor.Take(take); descriptor.Size(take); } - + + if (queryAggregator.After != null) + { + descriptor.SearchAfter(queryAggregator.After); + } + if (queryAggregator.Query != null) { descriptor.Query(q => queryAggregator.Query); @@ -103,8 +108,7 @@ public IEnumerable ExecuteCollection(QueryModel queryModel) { foreach (var orderByExpression in queryAggregator.OrderByExpressions) { - var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName) + - orderByExpression.GetKeywordIfNecessary(); + var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName); d.Field(property, orderByExpression.OrderingDirection == OrderingDirection.Asc ? SortOrder.Ascending diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs index 16072056..074c5bd8 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs @@ -1,7 +1,7 @@ using System.Linq.Expressions; +using AElf.EntityMapping.Linq; using Nest; using Remotion.Linq; -using Remotion.Linq.Parsing.Structure; using Volo.Abp.Domain.Entities; namespace AElf.EntityMapping.Elasticsearch.Linq @@ -11,7 +11,8 @@ public class ElasticsearchQueryable : QueryableBase, IElasticsearchQueryab { public ElasticsearchQueryable(IElasticClient elasticClient, ICollectionNameProvider collectionNameProvider, string index) - : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), QueryParser.CreateDefault(), + : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), + QueryParserFactory.Create(), new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index))) { } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryAggregator.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryAggregator.cs index 5a82752d..05524841 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/QueryAggregator.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/QueryAggregator.cs @@ -11,6 +11,7 @@ public class QueryAggregator public List PropertiesToSelect = new List(); public List OrderByExpressions = new List(); public List GroupByExpressions = new List(); + public object[] After { get; set; } } public class OrderProperties diff --git a/src/AElf.EntityMapping/AElf.EntityMapping.csproj b/src/AElf.EntityMapping/AElf.EntityMapping.csproj index e60f85d4..21c00ccd 100644 --- a/src/AElf.EntityMapping/AElf.EntityMapping.csproj +++ b/src/AElf.EntityMapping/AElf.EntityMapping.csproj @@ -10,7 +10,9 @@ + + diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index cc26451d..dcf19bb4 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -1,3 +1,4 @@ +using System.Linq.Dynamic.Core; using System.Linq.Expressions; using System.Transactions; using AElf.EntityMapping.Elasticsearch.Entities; @@ -789,4 +790,67 @@ public static string CreateBlockHash() { return Guid.NewGuid().ToString("N") + Guid.NewGuid().ToString("N"); } + + [Fact] + public async Task After_Test() + { + var blockIndex = new BlockIndex + { + Id = "block010", + BlockHash = "BlockHash", + BlockHeight = 10, + BlockTime = DateTime.Now, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + + blockIndex = new BlockIndex + { + Id = "block005", + BlockHash = "BlockHash", + BlockHeight = 10, + BlockTime = DateTime.Now, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + + blockIndex = new BlockIndex + { + Id = "block009", + BlockHash = "BlockHash", + BlockHeight = 9, + BlockTime = DateTime.Now, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + + blockIndex = new BlockIndex + { + Id = "block008", + BlockHash = "BlockHash", + BlockHeight = 12, + BlockTime = DateTime.Now, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + + blockIndex = new BlockIndex + { + Id = "block001", + BlockHash = "BlockHash", + BlockHeight = 1, + BlockTime = DateTime.Now, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + + + var queryable = await _elasticsearchRepository.GetQueryableAsync(); + queryable = queryable.Where(o => o.ChainId == "AELF" && o.BlockHeight >= 1).OrderBy(o=>o.BlockHeight).OrderBy(o=>o.Id).After(new object[]{9,"block009"}); + var list = queryable.ToList(); + list.Count.ShouldBe(3); + list[0].Id.ShouldBe("block005"); + list[1].Id.ShouldBe("block010"); + list[2].Id.ShouldBe("block008"); + } } \ No newline at end of file From 5e5e37f319db09cb47abfa37592ecd141c685fda Mon Sep 17 00:00:00 2001 From: zhxymh Date: Mon, 13 May 2024 17:21:14 +0800 Subject: [PATCH 14/20] feat: support search after --- .../Linq/AfterExpressionNode.cs | 36 +++++ .../Linq/AfterResultOperator.cs | 57 ++++++++ .../Linq/EntityMappingCachedReflectionInfo.cs | 12 ++ ...ntityMappingEvaluatableExpressionFilter.cs | 7 + .../Linq/EntityMappingNodeTypeProvider.cs | 135 ++++++++++++++++++ .../Linq/QueryParserFactory.cs | 27 ++++ .../Linq/QueryableExtensions.cs | 19 +++ 7 files changed, 293 insertions(+) create mode 100644 src/AElf.EntityMapping/Linq/AfterExpressionNode.cs create mode 100644 src/AElf.EntityMapping/Linq/AfterResultOperator.cs create mode 100644 src/AElf.EntityMapping/Linq/EntityMappingCachedReflectionInfo.cs create mode 100644 src/AElf.EntityMapping/Linq/EntityMappingEvaluatableExpressionFilter.cs create mode 100644 src/AElf.EntityMapping/Linq/EntityMappingNodeTypeProvider.cs create mode 100644 src/AElf.EntityMapping/Linq/QueryParserFactory.cs create mode 100644 src/AElf.EntityMapping/Linq/QueryableExtensions.cs diff --git a/src/AElf.EntityMapping/Linq/AfterExpressionNode.cs b/src/AElf.EntityMapping/Linq/AfterExpressionNode.cs new file mode 100644 index 00000000..de8faab3 --- /dev/null +++ b/src/AElf.EntityMapping/Linq/AfterExpressionNode.cs @@ -0,0 +1,36 @@ +using System.Collections.ObjectModel; +using System.Linq.Expressions; +using System.Reflection; +using Remotion.Linq.Clauses; +using Remotion.Linq.Parsing.Structure.IntermediateModel; + +namespace AElf.EntityMapping.Linq; + +public class AfterExpressionNode : ResultOperatorExpressionNodeBase +{ + private static readonly IEnumerable SupportedMethods = + new ReadOnlyCollection((typeof(QueryableExtensions).GetRuntimeMethods()).ToList()) + .Where((Func)(mi => mi.Name == "After")); + + public static IEnumerable GetSupportedMethods() => SupportedMethods; + + public AfterExpressionNode(MethodCallExpressionParseInfo parseInfo, Expression position) + : base(parseInfo, null, null) + { + Position = position; + } + + public Expression Position { get; } + + public override Expression Resolve( + ParameterExpression inputParameter, Expression expressionToBeResolved, + ClauseGenerationContext clauseGenerationContext) + { + return Source.Resolve(inputParameter, expressionToBeResolved, clauseGenerationContext); + } + + protected override ResultOperatorBase CreateResultOperator(ClauseGenerationContext clauseGenerationContext) + { + return new AfterResultOperator(Position); + } +} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Linq/AfterResultOperator.cs b/src/AElf.EntityMapping/Linq/AfterResultOperator.cs new file mode 100644 index 00000000..7fbe609f --- /dev/null +++ b/src/AElf.EntityMapping/Linq/AfterResultOperator.cs @@ -0,0 +1,57 @@ +using System.Linq.Expressions; +using Remotion.Linq.Clauses; +using Remotion.Linq.Clauses.ResultOperators; +using Remotion.Linq.Clauses.StreamedData; + +namespace AElf.EntityMapping.Linq; + +public class AfterResultOperator : SequenceTypePreservingResultOperatorBase +{ + private Expression _position; + + public AfterResultOperator(Expression position) + { + Position = position; + } + + public Expression Position + { + get => _position; + set + { + if (value.Type != typeof(object[])) + { + var message = + $"The value expression returns '{value.Type}', an expression returning 'System.Object[]' was expected."; + throw new ArgumentException(message, nameof(value)); + } + + _position = value; + } + } + + public object[] GetConstantPosition() + { + return GetConstantValueFromExpression("position", Position); + } + + public override ResultOperatorBase Clone(CloneContext cloneContext) + { + return new AfterResultOperator(Position); + } + + public override void TransformExpressions(Func transformation) + { + Position = transformation(Position); + } + + public override string ToString() + { + return "After(" + string.Join(",", _position) + ")"; + } + + public override StreamedSequence ExecuteInMemory(StreamedSequence input) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Linq/EntityMappingCachedReflectionInfo.cs b/src/AElf.EntityMapping/Linq/EntityMappingCachedReflectionInfo.cs new file mode 100644 index 00000000..b39ead10 --- /dev/null +++ b/src/AElf.EntityMapping/Linq/EntityMappingCachedReflectionInfo.cs @@ -0,0 +1,12 @@ +using System.Reflection; + +namespace System.Linq; + +public static class EntityMappingCachedReflectionInfo +{ + private static MethodInfo? _afterMethodInfo; + + public static MethodInfo AfterMethodInfo(Type source) => + (_afterMethodInfo ??= new Func, object[], IQueryable>(QueryableExtensions.After).GetMethodInfo().GetGenericMethodDefinition()) + .MakeGenericMethod(source); +} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Linq/EntityMappingEvaluatableExpressionFilter.cs b/src/AElf.EntityMapping/Linq/EntityMappingEvaluatableExpressionFilter.cs new file mode 100644 index 00000000..20d996fe --- /dev/null +++ b/src/AElf.EntityMapping/Linq/EntityMappingEvaluatableExpressionFilter.cs @@ -0,0 +1,7 @@ +using Remotion.Linq.Parsing.ExpressionVisitors.TreeEvaluation; + +namespace AElf.EntityMapping.Linq; + +public class EntityMappingEvaluatableExpressionFilter : EvaluatableExpressionFilterBase +{ +} diff --git a/src/AElf.EntityMapping/Linq/EntityMappingNodeTypeProvider.cs b/src/AElf.EntityMapping/Linq/EntityMappingNodeTypeProvider.cs new file mode 100644 index 00000000..7264a160 --- /dev/null +++ b/src/AElf.EntityMapping/Linq/EntityMappingNodeTypeProvider.cs @@ -0,0 +1,135 @@ +using System.Reflection; +using Remotion.Linq.Parsing.Structure; + +namespace AElf.EntityMapping.Linq; + +public class EntityMappingNodeTypeProvider : INodeTypeProvider +{ + public static EntityMappingNodeTypeProvider Create() + { + var typeProvider = new EntityMappingNodeTypeProvider(); + typeProvider.Register(AfterExpressionNode.GetSupportedMethods(), typeof(AfterExpressionNode)); + return typeProvider; + } + + private static readonly Dictionary> GenericMethodDefinitionCandidates = new(); + + public void Register(IEnumerable methods, Type nodeType) + { + foreach (var method in methods) + { + if (method.IsGenericMethod && !method.IsGenericMethodDefinition) + throw new InvalidOperationException(string.Format( + "Cannot register closed generic method '{0}', try to register its generic method definition instead.", + new object[1] + { + (object)method.Name + })); + if (method.DeclaringType.GetTypeInfo().IsGenericType && + !method.DeclaringType.GetTypeInfo().IsGenericTypeDefinition) + throw new InvalidOperationException(string.Format( + "Cannot register method '{0}' in closed generic type '{1}', try to register its equivalent in the generic type definition instead.", + new object[2] + { + (object)method.Name, + (object)method.DeclaringType + })); + _registeredMethodInfoTypes[method] = nodeType; + } + } + + public bool IsRegistered(MethodInfo method) + { + return GetNodeType(method) != null; + } + + public Type GetNodeType(MethodInfo method) + { + var methodDefinition = GetRegisterableMethodDefinition(method, throwOnAmbiguousMatch: false); + if (methodDefinition == null) + return null; + + Type result; + _registeredMethodInfoTypes.TryGetValue(methodDefinition, out result); + return result; + } + + public static MethodInfo GetRegisterableMethodDefinition(MethodInfo method, bool throwOnAmbiguousMatch) + { + var genericMethodDefinition = method.IsGenericMethod ? method.GetGenericMethodDefinition() : method; + if (!genericMethodDefinition.DeclaringType.GetTypeInfo().IsGenericType) + return genericMethodDefinition; + + Lazy candidates; + lock (GenericMethodDefinitionCandidates) + { + if (!GenericMethodDefinitionCandidates.TryGetValue(method, out candidates)) + { + candidates = + new Lazy(() => GetGenericMethodDefinitionCandidates(genericMethodDefinition)); + GenericMethodDefinitionCandidates.Add(method, candidates); + } + } + + if (candidates.Value.Length == 1) + return candidates.Value.Single(); + + if (!throwOnAmbiguousMatch) + return null; + + throw new NotSupportedException( + string.Format( + "A generic method definition cannot be resolved for method '{0}' on type '{1}' because a distinct match is not possible. " + + @"The method can still be registered using the following syntax: + +public static readonly NameBasedRegistrationInfo[] SupportedMethodNames = + new[] {{ + new NameBasedRegistrationInfo ( + ""{2}"", + mi => /* match rule based on MethodInfo */ + ) + }};", + method, + genericMethodDefinition.DeclaringType.GetGenericTypeDefinition(), + method.Name)); + } + + private readonly Dictionary _registeredMethodInfoTypes = new Dictionary(); + + public int RegisteredMethodInfoCount => _registeredMethodInfoTypes.Count; + + private static MethodInfo[] GetGenericMethodDefinitionCandidates(MethodInfo referenceMethodDefinition) + { + var declaringTypeDefinition = referenceMethodDefinition.DeclaringType.GetGenericTypeDefinition(); + + var referenceMethodSignature = + new[] { new { Name = "returnValue", Type = referenceMethodDefinition.ReturnType } } + .Concat(referenceMethodDefinition.GetParameters() + .Select(p => new { Name = p.Name, Type = p.ParameterType })) + .ToArray(); + + var candidates = declaringTypeDefinition.GetRuntimeMethods() + .Select( + m => new + { + Method = m, + SignatureNames = new[] { "returnValue" }.Concat(m.GetParameters().Select(p => p.Name)).ToArray(), + SignatureTypes = new[] { m.ReturnType }.Concat(m.GetParameters().Select(p => p.ParameterType)) + .ToArray() + }) + .Where(c => c.Method.Name == referenceMethodDefinition.Name && + c.SignatureTypes.Length == referenceMethodSignature.Length) + .ToArray(); + + for (var i = 0; i < referenceMethodSignature.Length; i++) + { + candidates = candidates + .Where(c => c.SignatureNames[i] == referenceMethodSignature[i].Name) + .Where(c => c.SignatureTypes[i] == referenceMethodSignature[i].Type || + c.SignatureTypes[i].GetTypeInfo().ContainsGenericParameters) + .ToArray(); + } + + return candidates.Select(c => c.Method).ToArray(); + } +} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Linq/QueryParserFactory.cs b/src/AElf.EntityMapping/Linq/QueryParserFactory.cs new file mode 100644 index 00000000..979bbc39 --- /dev/null +++ b/src/AElf.EntityMapping/Linq/QueryParserFactory.cs @@ -0,0 +1,27 @@ +using Remotion.Linq.Parsing.ExpressionVisitors.Transformation; +using Remotion.Linq.Parsing.Structure; +using Remotion.Linq.Parsing.Structure.NodeTypeProviders; + +namespace AElf.EntityMapping.Linq; + +public class QueryParserFactory +{ + public static QueryParser Create() + { + var transformerRegistry = ExpressionTransformerRegistry.CreateDefault(); + var evaluatableExpressionFilter = new EntityMappingEvaluatableExpressionFilter(); + + var innerProviders = new INodeTypeProvider[] + { + MethodInfoBasedNodeTypeRegistry.CreateFromRelinqAssembly(), + MethodNameBasedNodeTypeRegistry.CreateFromRelinqAssembly(), + EntityMappingNodeTypeProvider.Create() + }; + var nodeType = new CompoundNodeTypeProvider (innerProviders); + + var expressionTreeParser = new ExpressionTreeParser ( + nodeType, + ExpressionTreeParser.CreateDefaultProcessor (transformerRegistry, evaluatableExpressionFilter)); + return new QueryParser(expressionTreeParser); + } +} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Linq/QueryableExtensions.cs b/src/AElf.EntityMapping/Linq/QueryableExtensions.cs new file mode 100644 index 00000000..c5058c05 --- /dev/null +++ b/src/AElf.EntityMapping/Linq/QueryableExtensions.cs @@ -0,0 +1,19 @@ +using System.Linq.Expressions; + +namespace System.Linq; + +public static class QueryableExtensions +{ + public static IQueryable After(this IQueryable source, object[] position) + { + ArgumentNullException.ThrowIfNull(source); + ArgumentNullException.ThrowIfNull(position); + + return source.Provider.CreateQuery( + Expression.Call( + null, + EntityMappingCachedReflectionInfo.AfterMethodInfo(typeof(TSource)), + source.Expression, Expression.Constant(position) + )); + } +} \ No newline at end of file From 1236e21c37073a280738f7866b5b24e641d20aef Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 21 May 2024 10:55:56 +0800 Subject: [PATCH 15/20] fix: sub object order by & nested object order by query --- .../IElasticsearchClientProvider.cs | 16 ++++++++- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 35 +++++++++++++++++-- .../Linq/ElasticsearchQueryExecutor.cs | 6 ++-- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs index 988f51df..11717bc1 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs @@ -1,3 +1,4 @@ +using System.Text; using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Microsoft.Extensions.Options; @@ -19,7 +20,20 @@ public ElasticsearchClientProvider(IOptions options) { var uris = options.Value.Uris.ConvertAll(x => new Uri(x)); var connectionPool = new StaticConnectionPool(uris); - var settings = new ConnectionSettings(connectionPool); + var settings = new ConnectionSettings(connectionPool).DisableDirectStreaming(); + // .OnRequestCompleted(callDetails => + // { + // // Print Request DSL + // if (callDetails.RequestBodyInBytes != null) + // { + // Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); + // } + // // // Print Response Data + // // if (callDetails.ResponseBodyInBytes != null) + // // { + // // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); + // // } + // }); _elasticClient = new ElasticClient(settings); } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index f8a1c70a..1b909644 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -133,12 +133,41 @@ public override void VisitOrderByClause(OrderByClause orderByClause, QueryModel { var memberExpression = (MemberExpression)ordering.Expression; var direction = orderByClause.Orderings[0].OrderingDirection; - var propertyName = GetFullNameKey(memberExpression); - var type = memberExpression.Type; - QueryAggregator.OrderByExpressions.Add(new OrderProperties(propertyName, type, direction)); + //get full property path if there is sub object + string propertyName = GetFullPropertyPath(memberExpression); + // string propertyName = GetFullNameKey(memberExpression); + + if (!string.IsNullOrEmpty(propertyName)) + { + var type = memberExpression.Type; + QueryAggregator.OrderByExpressions.Add(new OrderProperties(propertyName, type, direction)); + } } base.VisitOrderByClause(orderByClause, queryModel, index); } + + private string GetFullPropertyPath(Expression expression) + { + switch (expression) + { + case MemberExpression memberExpression: + var parentPath = GetFullPropertyPath(memberExpression.Expression); + var currentMemberName = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); + return string.IsNullOrEmpty(parentPath) ? currentMemberName : $"{parentPath}.{currentMemberName}"; + + case MethodCallExpression methodCallExpression: + // Handles method calls like 'get_Item', which are usually associated with indexed access to collections + if (methodCallExpression.Method.Name.Equals("get_Item") && methodCallExpression.Object != null) + { + // Assuming this is an indexed access to an array or list, we will ignore the index and use only the name of the collection + var collectionPath = GetFullPropertyPath(methodCallExpression.Object); + return collectionPath; // Returns the path of the collection directly, without adding an index + } + break; + } + + return null; + } } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index 9ee8fbf0..e6faa1a3 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -103,8 +103,10 @@ public IEnumerable ExecuteCollection(QueryModel queryModel) { foreach (var orderByExpression in queryAggregator.OrderByExpressions) { - var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName) + - orderByExpression.GetKeywordIfNecessary(); + // var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName) + + // orderByExpression.GetKeywordIfNecessary(); + var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName); + d.Field(property, orderByExpression.OrderingDirection == OrderingDirection.Asc ? SortOrder.Ascending From f21dbbd85600507f90c7ee1bddbf3100e72839ff Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 21 May 2024 11:01:51 +0800 Subject: [PATCH 16/20] perf: optimize code --- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 1 - .../Linq/ElasticsearchQueryExecutor.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index 1b909644..25b7b195 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -135,7 +135,6 @@ public override void VisitOrderByClause(OrderByClause orderByClause, QueryModel var direction = orderByClause.Orderings[0].OrderingDirection; //get full property path if there is sub object string propertyName = GetFullPropertyPath(memberExpression); - // string propertyName = GetFullNameKey(memberExpression); if (!string.IsNullOrEmpty(propertyName)) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index e6faa1a3..90f6a47e 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -103,8 +103,6 @@ public IEnumerable ExecuteCollection(QueryModel queryModel) { foreach (var orderByExpression in queryAggregator.OrderByExpressions) { - // var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName) + - // orderByExpression.GetKeywordIfNecessary(); var property = _propertyNameInferrerParser.Parser(orderByExpression.PropertyName); d.Field(property, From 15e8b50e8fab9b59802027b2dfa44571ca73c279 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 21 May 2024 11:10:13 +0800 Subject: [PATCH 17/20] perf: optimize code --- .../IElasticsearchClientProvider.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs index 11717bc1..89b33684 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs @@ -20,7 +20,8 @@ public ElasticsearchClientProvider(IOptions options) { var uris = options.Value.Uris.ConvertAll(x => new Uri(x)); var connectionPool = new StaticConnectionPool(uris); - var settings = new ConnectionSettings(connectionPool).DisableDirectStreaming(); + var settings = new ConnectionSettings(connectionPool); + // .DisableDirectStreaming(); // .OnRequestCompleted(callDetails => // { // // Print Request DSL From 18a88d260eb9434cde36a0f42fff1d2d5cafab68 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 21 May 2024 11:33:57 +0800 Subject: [PATCH 18/20] fix: GetCollectionName unit test --- .../ElasticsearchCollectionNameProviderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs index 532bc492..fcbc73ef 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs @@ -72,7 +72,7 @@ public async Task GetCollectionName_Test() Type = ConditionType.Equal }); collectionNames = await _collectionNameProvider.GetFullCollectionNameAsync(collectionNameCondition); - collectionNames.Count.ShouldBe(0); + collectionNames.Count.ShouldBe(1); } [Fact] From 7a7f79805bd44a7cf7fb470ceab371251187843e Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Tue, 21 May 2024 15:05:27 +0800 Subject: [PATCH 19/20] fix: get collection name unit test --- .../ElasticsearchCollectionNameProviderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs index 532bc492..fcbc73ef 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs @@ -72,7 +72,7 @@ public async Task GetCollectionName_Test() Type = ConditionType.Equal }); collectionNames = await _collectionNameProvider.GetFullCollectionNameAsync(collectionNameCondition); - collectionNames.Count.ShouldBe(0); + collectionNames.Count.ShouldBe(1); } [Fact] From e8a631bed0ea87a19232c324bfb1dd786ab7bbf6 Mon Sep 17 00:00:00 2001 From: "bourne.syb" <109585311+AElfBourneShi@users.noreply.github.com> Date: Wed, 7 Aug 2024 11:56:51 +0800 Subject: [PATCH 20/20] Feature: delete index (#25) * feat: delete index * fix: sub object query of Multiple levels * feat: optimize index delete method * fix: recover unit test * feat: add GitHub Action for NuGet publish * feat: add GitHub Action for unit test and code coverage --------- Co-authored-by: jason-aelf --- .github/workflows/nuget-publish.yml | 40 ++++++++++++++ .github/workflows/test-with-code-coverage.yml | 55 +++++++++++++++++++ .../Linq/ElasticsearchQueryExecutor.cs | 2 +- .../Linq/GeneratorExpressionTreeVisitor.cs | 47 ++++++++++++---- .../Services/ElasticIndexService.cs | 37 +++++++++++++ .../Services/IElasticIndexService.cs | 1 + .../IEntityMappingBasicRepository.cs | 2 + .../AElfElasticsearchTestsModule.cs | 6 +- .../Entities/AccountBalanceEntity.cs | 13 +++++ .../Entities/AeFinderEntity.cs | 32 +++++++++++ .../Entities/Metadata.cs | 19 +++++++ .../ElasticsearchRepositoryTests.cs | 39 +++++++++++++ 12 files changed, 279 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/nuget-publish.yml create mode 100644 .github/workflows/test-with-code-coverage.yml create mode 100644 test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AccountBalanceEntity.cs create mode 100644 test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AeFinderEntity.cs create mode 100644 test/AElf.EntityMapping.Elasticsearch.Tests/Entities/Metadata.cs diff --git a/.github/workflows/nuget-publish.yml b/.github/workflows/nuget-publish.yml new file mode 100644 index 00000000..ed46252b --- /dev/null +++ b/.github/workflows/nuget-publish.yml @@ -0,0 +1,40 @@ +name: Publish NuGet Package + +on: + push: + tags: + - "v*.*.*" + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + run: dotnet restore + + - name: Get the version from git tags + id: get_version + run: | + TAG=$(git describe --tags --abbrev=0) + VERSION=${TAG#v} + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Build the project + run: dotnet build --configuration Release --no-restore + + - name: Pack the NuGet package + run: dotnet pack --configuration Release --no-build --output ./nupkg -p:PackageVersion=${{ env.VERSION }} + + - name: Publish the NuGet package + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + run: dotnet nuget push ./nupkg/*.nupkg --api-key $NUGET_API_KEY --source https://api.nuget.org/v3/index.json diff --git a/.github/workflows/test-with-code-coverage.yml b/.github/workflows/test-with-code-coverage.yml new file mode 100644 index 00000000..d1d39f5a --- /dev/null +++ b/.github/workflows/test-with-code-coverage.yml @@ -0,0 +1,55 @@ +name: Test with code coverage + +on: + push: + branches: + - '**' + +env: + DOTNET_INSTALL_DIR: "./.dotnet" + +jobs: + test: + runs-on: ubuntu-20.04 + permissions: + pull-requests: write + contents: write + services: + elasticsearch: + image: elasticsearch:7.17.0 + ports: + - 9200:9200 + options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '7.0' + - name: Verify Elasticsearch connection + env: + ELASTIC_SEARCH_URL: http://127.0.0.1:${{ job.services.elasticsearch.ports[9200] }} + run: | + echo $ELASTIC_SEARCH_URL + curl -fsSL "$ELASTIC_SEARCH_URL/_cat/health?h=status" + - name: Install dependencies + run: dotnet restore --verbosity quiet + + - name: Build + run: dotnet build --no-restore /clp:ErrorsOnly /p:GeneratePackageOnBuild=false --verbosity quiet + + - name: Test + run: | + for name in `ls ./test/*.Tests/*.csproj | awk '{print $NF}'`; + do + dotnet test ${name} --no-restore --no-build --logger trx --settings CodeCoverage.runsettings --results-directory coverage --collect:"XPlat Code Coverage" + done + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + files: coverage/*/coverage.cobertura.xml + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index 8e93a67e..e6cfe552 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -144,7 +144,7 @@ public IEnumerable ExecuteCollection(QueryModel queryModel) }); } - + // var dsl = _elasticClient.RequestResponseSerializer.SerializeToString(descriptor); return descriptor; }); diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 01771c0f..7aa1750a 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -127,22 +127,45 @@ protected override Expression VisitMethodCall(MethodCallExpression expression) return expression; } - private string GetFullNameKey(MemberExpression memberExpression) + // private string GetFullNameKey(MemberExpression memberExpression) + // { + // var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); + // while (memberExpression.Expression != null) + // { + // memberExpression = memberExpression.Expression as MemberExpression; + // if (memberExpression == null) + // { + // break; + // } + // + // key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); + // return key; + // } + // + // return key; + // } + + private string GetFullPropertyPath(Expression expression) { - var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); - while (memberExpression.Expression != null) + switch (expression) { - memberExpression = memberExpression.Expression as MemberExpression; - if (memberExpression == null) - { - break; - } + case MemberExpression memberExpression: + var parentPath = GetFullPropertyPath(memberExpression.Expression); + var currentMemberName = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); + return string.IsNullOrEmpty(parentPath) ? currentMemberName : $"{parentPath}.{currentMemberName}"; - key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); - return key; + case MethodCallExpression methodCallExpression: + // Handles method calls like 'get_Item', which are usually associated with indexed access to collections + if (methodCallExpression.Method.Name.Equals("get_Item") && methodCallExpression.Object != null) + { + // Assuming this is an indexed access to an array or list, we will ignore the index and use only the name of the collection + var collectionPath = GetFullPropertyPath(methodCallExpression.Object); + return collectionPath; // Returns the path of the collection directly, without adding an index + } + break; } - return key; + return null; } protected override Expression VisitMember(MemberExpression expression) @@ -150,7 +173,7 @@ protected override Expression VisitMember(MemberExpression expression) Visit(expression.Expression); PropertyType = Nullable.GetUnderlyingType(expression.Type) ?? expression.Type; - PropertyName = _propertyNameInferrerParser.Parser(GetFullNameKey(expression)); + PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(expression)); // Implicit boolean is only a member visit if (expression.Type == typeof(bool)) diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs index 454c7720..d20e916e 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs @@ -146,4 +146,41 @@ public async Task CreateCollectionRouteKeyIndexAsync(Type type, int shard = 1, i } } } + + public async Task DeleteIndexAsync(string collectionName = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(collectionName)) + { + throw new ArgumentNullException(nameof(collectionName), "Collection name must be provided."); + } + + try + { + var elasticClient = await GetElasticsearchClientAsync(); + var response = await elasticClient.Indices.DeleteAsync(collectionName, ct: cancellationToken); + if (!response.IsValid) + { + if (response.ServerError == null) + { + return; + } + + if (response.ServerError?.Status == 404) + { + _logger.LogError("Failed to delete index {0} does not exist.", collectionName); + return; + } + + // Log the error or throw an exception based on the response + throw new ElasticsearchException($"Failed to delete index {collectionName}: {response.ServerError.Error.Reason}"); + } + + _logger.LogInformation("Index {0} deleted successfully.", collectionName); + } + catch (Exception ex) + { + // Handle exceptions from the client (network issues, etc.) + throw new ElasticsearchException($"An error occurred while delete index {collectionName}: {ex.Message}"); + } + } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs index 6136c3f6..b4b9f702 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs @@ -10,4 +10,5 @@ Task CreateIndexTemplateAsync(string indexTemplateName, string indexName, Type i Task CreateCollectionRouteKeyIndexAsync(Type indexEntityType, int numberOfShards, int numberOfReplicas); + Task DeleteIndexAsync(string collectionName = null, CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/src/AElf.EntityMapping/Repositories/IEntityMappingBasicRepository.cs b/src/AElf.EntityMapping/Repositories/IEntityMappingBasicRepository.cs index a3db3510..fdea54c1 100644 --- a/src/AElf.EntityMapping/Repositories/IEntityMappingBasicRepository.cs +++ b/src/AElf.EntityMapping/Repositories/IEntityMappingBasicRepository.cs @@ -19,5 +19,7 @@ public interface IEntityMappingBasicRepository where TEntity : cl Task DeleteAsync(TEntity model, string collectionName = null, CancellationToken cancellationToken = default); Task DeleteManyAsync(List list, string collectionName = null, CancellationToken cancellationToken = default); + + } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/AElfElasticsearchTestsModule.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/AElfElasticsearchTestsModule.cs index 596c4ce8..2a4d9dd0 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/AElfElasticsearchTestsModule.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/AElfElasticsearchTestsModule.cs @@ -1,4 +1,5 @@ using AElf.EntityMapping.Elasticsearch.Options; +using AElf.EntityMapping.Elasticsearch.Services; using AElf.EntityMapping.Options; using AElf.EntityMapping.TestBase; using Elasticsearch.Net; @@ -6,6 +7,7 @@ using Microsoft.Extensions.Options; using Volo.Abp; using Volo.Abp.Modularity; +using Volo.Abp.Threading; namespace AElf.EntityMapping.Elasticsearch; @@ -36,9 +38,11 @@ public override void OnApplicationShutdown(ApplicationShutdownContext context) var clientProvider = context.ServiceProvider.GetRequiredService(); var client = clientProvider.GetClient(); + var elasticIndexService = context.ServiceProvider.GetRequiredService(); var indexPrefix = option.Value.CollectionPrefix.ToLower(); - client.Indices.Delete(indexPrefix+"*"); + // client.Indices.Delete(indexPrefix+"*"); + AsyncHelper.RunSync(async () => await elasticIndexService.DeleteIndexAsync(indexPrefix+"*")); client.Indices.DeleteTemplate(indexPrefix + "*"); } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AccountBalanceEntity.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AccountBalanceEntity.cs new file mode 100644 index 00000000..106176b0 --- /dev/null +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AccountBalanceEntity.cs @@ -0,0 +1,13 @@ +using AElf.EntityMapping.Entities; +using Nest; + +namespace AElf.EntityMapping.Elasticsearch.Entities; + +public class AccountBalanceEntity: AeFinderEntity, IEntityMappingEntity +{ + [Keyword]public override string Id { get; set; } + public Metadata Metadata { get; set; } = new (); + [Keyword] public string Account { get; set; } + [Keyword] public string Symbol { get; set; } + public long Amount { get; set; } +} \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AeFinderEntity.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AeFinderEntity.cs new file mode 100644 index 00000000..022c389b --- /dev/null +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/AeFinderEntity.cs @@ -0,0 +1,32 @@ +using AElf.EntityMapping.Entities; +using Nest; +using Volo.Abp.Domain.Entities; + +namespace AElf.EntityMapping.Elasticsearch.Entities; + +public abstract class AeFinderEntity:Entity,IEntity +{ + /// + public virtual TKey Id { get; set; } + + protected AeFinderEntity() + { + + } + + protected AeFinderEntity(TKey id) + { + Id = id; + } + + public override object[] GetKeys() + { + return new object[] {Id}; + } + + /// + public override string ToString() + { + return $"[ENTITY: {GetType().Name}] Id = {Id}"; + } +} \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/Metadata.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/Metadata.cs new file mode 100644 index 00000000..10692424 --- /dev/null +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/Metadata.cs @@ -0,0 +1,19 @@ +using Nest; + +namespace AElf.EntityMapping.Elasticsearch.Entities; + +public class Metadata +{ + [Keyword] + public string ChainId { get; set; } + public BlockMetadata Block { get; set; } + public bool IsDeleted { get; set; } +} + +public class BlockMetadata +{ + [Keyword] + public string BlockHash { get; set; } + public long BlockHeight { get; set; } + public DateTime BlockTime { get; set; } +} \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index dcf19bb4..90cbe5e9 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -14,6 +14,7 @@ public class ElasticsearchRepositoryTests : AElfElasticsearchTestBase { private readonly IElasticsearchRepository _elasticsearchRepository; private readonly IElasticsearchRepository _transactionIndexRepository; + private readonly IElasticsearchRepository _accountBalanceRepository; private readonly IElasticIndexService _elasticIndexService; private readonly AElfEntityMappingOptions _option; @@ -21,6 +22,7 @@ public ElasticsearchRepositoryTests() { _elasticsearchRepository = GetRequiredService>(); _transactionIndexRepository = GetRequiredService>(); + _accountBalanceRepository = GetRequiredService>(); _elasticIndexService = GetRequiredService(); _option = GetRequiredService>().Value; } @@ -591,6 +593,43 @@ public async Task GetList_Nested_Test() filterList.Count.ShouldBe(1); } + [Fact] + public async Task SubObjectQueryTest() + { + for (int i = 1; i <= 7; i++) + { + var accountBalanceEntity = new AccountBalanceEntity + { + Id = "block" + i, + Account = "BlockHash" + i, + Amount = i, + Symbol = "AELF", + Metadata = new Metadata() + { + ChainId = "tDVV", + Block=new BlockMetadata() + { + BlockHash = "BlockHash" + i, + BlockHeight = i, + BlockTime = DateTime.Now.AddDays(-10 + i) + }, + IsDeleted=false + } + }; + await _accountBalanceRepository.AddAsync(accountBalanceEntity); + } + + var queryable = await _accountBalanceRepository.GetQueryableAsync(); + var list1 = queryable.Where(o => o.Metadata.Block.BlockHash == "BlockHash5").ToList(); + // var list1 = queryable.Where(o => o.Metadata.ChainId == "tDVV").ToList(); + // var list1 = queryable.Where(o => o.Account == "BlockHash3").ToList(); + list1.Count.ShouldBe(1); + + var list = await _accountBalanceRepository.GetListAsync(o=>o.Metadata.Block.BlockHash == "BlockHash4"); + // var list = await _accountBalanceRepository.GetListAsync(o => o.Account == "BlockHash3"); + list.Count.ShouldBe(1); + } + [Fact] public async Task GetList_MultipleChain_Test()