Skip to content

Commit

Permalink
Merge pull request #38 from AElfProject/release/1.3.0
Browse files Browse the repository at this point in the history 
Release v1.3.0
  • Loading branch information
@jason-aelf
jason-aelf authored Oct 22, 2024
2 parents a620ce3 + 7d4e68f commit 664d942
Show file tree
Hide file tree
Showing 9 changed files with 279 additions and 48 deletions.
17 changes: 16 additions & 1 deletion src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,21 @@ public class ElasticsearchResponseHelper
{
public static string GetErrorMessage(IResponse response)
{
return response.ServerError == null ? "Unknown error." : response.ServerError.ToString();
if (response.ServerError == null)
{
if (response.OriginalException == null)
{
return "Unknown error.";
}

if (response.OriginalException.InnerException == null)
{
return response.OriginalException.Message;
}

return response.OriginalException.InnerException.Message;
}

return response.ServerError.ToString();
}
}
2 changes: 1 addition & 1 deletion src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public ElasticsearchClientProvider(IOptions<ElasticsearchOptions> options)
var uris = options.Value.Uris.ConvertAll(x => new Uri(x));
var connectionPool = new StaticConnectionPool(uris);
var settings = new ConnectionSettings(connectionPool);
// .DisableDirectStreaming();
// .DisableDirectStreaming()
// .OnRequestCompleted(callDetails =>
// {
// // Print Request DSL
Expand Down
12 changes: 9 additions & 3 deletions src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using AElf.EntityMapping.Elasticsearch.Linq;
using AElf.EntityMapping.Elasticsearch.Options;
using Microsoft.Extensions.Options;
using Nest;
using Volo.Abp.Domain.Entities;

Expand All @@ -14,14 +16,18 @@ public class ElasticsearchQueryableFactory<TEntity> : IElasticsearchQueryableFac
where TEntity : class, IEntity
{
private readonly ICollectionNameProvider<TEntity> _collectionNameProvider;
private readonly ElasticsearchOptions _elasticsearchOptions;

public ElasticsearchQueryableFactory(ICollectionNameProvider<TEntity> collectionNameProvider)
public ElasticsearchQueryableFactory(ICollectionNameProvider<TEntity> collectionNameProvider,
IOptions<ElasticsearchOptions> elasticsearchOptions)
{
_collectionNameProvider = collectionNameProvider;
_elasticsearchOptions = elasticsearchOptions.Value;
}

public ElasticsearchQueryable<TEntity> Create(IElasticClient client, string index = null)
public ElasticsearchQueryable<TEntity> Create(IElasticClient client,
string index = null)
{
return new ElasticsearchQueryable<TEntity>(client, _collectionNameProvider, index);
return new ElasticsearchQueryable<TEntity>(client, _collectionNameProvider, index, _elasticsearchOptions);
}
}
10 changes: 8 additions & 2 deletions src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
using System.Collections.ObjectModel;
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Options;
using AElf.EntityMapping.Linq;
using AElf.EntityMapping.Options;
using Microsoft.Extensions.Options;
using Nest;
using Remotion.Linq;
using Remotion.Linq.Clauses;
Expand All @@ -13,12 +16,15 @@ public class ElasticsearchGeneratorQueryModelVisitor<TU> : QueryModelVisitorBase
{
private readonly PropertyNameInferrerParser _propertyNameInferrerParser;
private readonly INodeVisitor _nodeVisitor;
private readonly ElasticsearchOptions _elasticsearchOptions;
private QueryAggregator QueryAggregator { get; set; } = new QueryAggregator();

public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser)
public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser,
ElasticsearchOptions elasticsearchOptions)
{
_propertyNameInferrerParser = propertyNameInferrerParser;
_nodeVisitor = new NodeVisitor();
_elasticsearchOptions = elasticsearchOptions;
}

public QueryAggregator GenerateElasticQuery<T>(QueryModel queryModel)
Expand Down Expand Up @@ -48,7 +54,7 @@ public override void VisitMainFromClause(MainFromClause fromClause, QueryModel q

public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index)
{
var tree = new GeneratorExpressionTreeVisitor<TU>(_propertyNameInferrerParser);
var tree = new GeneratorExpressionTreeVisitor<TU>(_propertyNameInferrerParser, _elasticsearchOptions);
tree.Visit(whereClause.Predicate);
if (QueryAggregator.Query == null)
{
Expand Down
9 changes: 7 additions & 2 deletions src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Dynamic;
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Exceptions;
using AElf.EntityMapping.Elasticsearch.Options;
using Elasticsearch.Net;
using Nest;
using Newtonsoft.Json;
Expand All @@ -23,16 +24,20 @@ public class ElasticsearchQueryExecutor<TEntity>: IQueryExecutor
private readonly JsonSerializerSettings _deserializerSettings;
private readonly ICollectionNameProvider<TEntity> _collectionNameProvider;
private const int ElasticQueryLimit = 10000;
private readonly ElasticsearchOptions _elasticsearchOptions;

public ElasticsearchQueryExecutor(IElasticClient elasticClient,
ICollectionNameProvider<TEntity> collectionNameProvider, string index)
ICollectionNameProvider<TEntity> collectionNameProvider, string index,
ElasticsearchOptions elasticsearchOptions)
{
_elasticClient = elasticClient;
_collectionNameProvider = collectionNameProvider;
_index = index;
_propertyNameInferrerParser = new PropertyNameInferrerParser(_elasticClient);
_elasticsearchOptions = elasticsearchOptions;
_elasticsearchGeneratorQueryModelVisitor =
new ElasticsearchGeneratorQueryModelVisitor<TEntity>(_propertyNameInferrerParser);
new ElasticsearchGeneratorQueryModelVisitor<TEntity>(_propertyNameInferrerParser,
_elasticsearchOptions);
_deserializerSettings = new JsonSerializerSettings
{
// Nest maps TimeSpan as a long (TimeSpan ticks)
Expand Down
7 changes: 4 additions & 3 deletions src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Options;
using AElf.EntityMapping.Linq;
using Nest;
using Remotion.Linq;
Expand All @@ -10,10 +11,10 @@ public class ElasticsearchQueryable<T> : QueryableBase<T>, IElasticsearchQueryab
where T : class, IEntity
{
public ElasticsearchQueryable(IElasticClient elasticClient, ICollectionNameProvider<T> collectionNameProvider,
string index)
: base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>),
string index, ElasticsearchOptions elasticsearchOptions)
: base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>),
QueryParserFactory.Create(),
new ElasticsearchQueryExecutor<T>(elasticClient, collectionNameProvider, index)))
new ElasticsearchQueryExecutor<T>(elasticClient, collectionNameProvider, index, elasticsearchOptions)))
{
}

Expand Down
179 changes: 143 additions & 36 deletions src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Linq.Expressions;
using AElf.EntityMapping.Elasticsearch.Options;
using Elasticsearch.Net;
using Remotion.Linq.Clauses;
using Remotion.Linq.Clauses.Expressions;
Expand All @@ -10,6 +11,7 @@ namespace AElf.EntityMapping.Elasticsearch.Linq
public class GeneratorExpressionTreeVisitor<T> : ThrowingExpressionVisitor
{
private readonly PropertyNameInferrerParser _propertyNameInferrerParser;
private readonly ElasticsearchOptions _elasticsearchOptions;

private object Value { get; set; }
private string PropertyName { get; set; }
Expand All @@ -19,9 +21,11 @@ public class GeneratorExpressionTreeVisitor<T> : ThrowingExpressionVisitor
public IDictionary<Expression, Node> QueryMap { get; } =
new Dictionary<Expression, Node>();

public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser)
public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser,
ElasticsearchOptions elasticsearchOptions)
{
_propertyNameInferrerParser = propertyNameInferrerParser;
_elasticsearchOptions = elasticsearchOptions;
}

protected override Expression VisitUnary(UnaryExpression expression)
Expand Down Expand Up @@ -129,22 +133,22 @@ protected override Expression VisitMethodCall(MethodCallExpression expression)

// private string GetFullNameKey(MemberExpression memberExpression)
// {
// var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name);
// while (memberExpression.Expression != null)
// {
// memberExpression = memberExpression.Expression as MemberExpression;
// if (memberExpression == null)
// {
// break;
// }
//
// key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key);
// return key;
// }
//
// return key;
// var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name);
// while (memberExpression.Expression != null)
// {
// memberExpression = memberExpression.Expression as MemberExpression;
// if (memberExpression == null)
// {
// break;
// }
//
// key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key);
// return key;
// }

//
// return key;
// }

private string GetFullPropertyPath(Expression expression)
{
switch (expression)
Expand All @@ -162,6 +166,7 @@ private string GetFullPropertyPath(Expression expression)
var collectionPath = GetFullPropertyPath(methodCallExpression.Object);
return collectionPath; // Returns the path of the collection directly, without adding an index
}

break;
}

Expand Down Expand Up @@ -203,15 +208,15 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
Visit(containsResultOperator.Item);
Visit(expression.QueryModel.MainFromClause.FromExpression);

if (containsResultOperator.Item.Type == typeof(Guid))
//Check if the number of items in the Terms query array within the Contains clause is too large.
if (expression.QueryModel.MainFromClause
.FromExpression is ConstantExpression constantExpression)
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid>)Value).Select(x => x.ToString()));
CheckTermsArrayLength(constantExpression);
}

if (containsResultOperator.Item.Type == typeof(Guid?))
{
query = new TermsNode(PropertyName, ((IEnumerable<Guid?>)Value).Select(x => x.ToString()));
}
// Handling different types
query = GetDifferentTypesTermsQueryNode();

QueryMap[expression] = ParseQuery(query);
break;
Expand All @@ -227,23 +232,32 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)

foreach (var whereClause in whereClauses)
{
Visit(whereClause.Predicate);
Node tmp = (Node)QueryMap[whereClause.Predicate].Clone();
QueryMap[expression] = tmp;
QueryMap[expression].IsSubQuery = true;
QueryMap[expression].SubQueryPath = from; //from.ToLower();
QueryMap[expression].SubQueryFullPath = fullPath;
// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true);
BinaryExpression predicate = (BinaryExpression)whereClause.Predicate;
if (predicate.Left is BinaryExpression)
if (whereClause.Predicate is SubQueryExpression subQueryExpression)
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true);
HandleNestedContains(subQueryExpression, expression, from,
fullPath);
}

if (predicate.Right is BinaryExpression)
else
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true);
Visit(whereClause.Predicate);
Node tmp = (Node)QueryMap[whereClause.Predicate].Clone();
QueryMap[expression] = tmp;
QueryMap[expression].IsSubQuery = true;
QueryMap[expression].SubQueryPath = from; //from.ToLower();
QueryMap[expression].SubQueryFullPath = fullPath;
// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true);
BinaryExpression predicate = (BinaryExpression)whereClause.Predicate;
if (predicate.Left is BinaryExpression)
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true);
}

if (predicate.Right is BinaryExpression)
{
VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true);
}
}

}

break;
Expand Down Expand Up @@ -281,6 +295,98 @@ protected override Expression VisitSubQuery(SubQueryExpression expression)
return expression;
}

private void HandleNestedContains(SubQueryExpression subQueryExpression, Expression expression,
string subQueryPath, string subQueryFullPath)
{
if (subQueryExpression == null || expression == null)
throw new ArgumentNullException("SubQueryExpression or expression cannot be null.");

//Check if the number of items in the Terms query array within the Contains clause is too large.
if (subQueryExpression.QueryModel.MainFromClause
.FromExpression is ConstantExpression constantExpression)
{
CheckTermsArrayLength(constantExpression);
}

foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators)
{
switch (resultOperator)
{
case ContainsResultOperator containsResultOperator:
Visit(containsResultOperator.Item);
Visit(subQueryExpression.QueryModel.MainFromClause.FromExpression);
break;
}
}

Node query;
query = GetDifferentTypesTermsQueryNode();

QueryMap[expression] = ParseQuery(query);
QueryMap[expression].IsSubQuery = true;
QueryMap[expression].SubQueryPath = subQueryPath; //from.ToLower();
QueryMap[expression].SubQueryFullPath = subQueryFullPath;
}

private Node GetDifferentTypesTermsQueryNode()
{
Node query;
if (PropertyType == typeof(Guid))
{
query = GetTermsNode<Guid>();
}
else if (PropertyType == typeof(int))
{
query = GetTermsNode<int>();
}
else if (PropertyType == typeof(long))
{
query = GetTermsNode<long>();
}
else if (PropertyType == typeof(double))
{
query = GetTermsNode<double>();
}
else if (PropertyType == typeof(bool))
{
query = GetTermsNode<bool>();
}
else if (PropertyType == typeof(DateTime))
{
query = new TermsNode(PropertyName,
((IEnumerable<DateTime>)Value).Select(x => x.ToString("o")));
}
else if (PropertyType == typeof(string))
{
query = new TermsNode(PropertyName, (IEnumerable<string>)Value);
}
else
{
throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Terms queries.");
}

return query;
}

private TermsNode GetTermsNode<T>()

Check warning on line 371 in src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs

View workflow job for this annotation

GitHub Actions / publish 

Type parameter 'T' has the same name as the type parameter from outer type 'GeneratorExpressionTreeVisitor<T>'
{
return new TermsNode(PropertyName,
((IEnumerable<T>)Value).Select(x => x.ToString()));
}

private void CheckTermsArrayLength(ConstantExpression constantExpression)
{
if (constantExpression.Value is System.Collections.IEnumerable objectList)
{
var count = objectList.Cast<object>().Count();
if (count > _elasticsearchOptions.TermsArrayMaxLength)
{
throw new ArgumentException(
$"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items.");
}
}
}

protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression)
{
return expression;
Expand Down Expand Up @@ -597,7 +703,8 @@ private object ConvertEnumValue(Type entityType, string propertyName, object val
return (int)enumValue;
}

protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, bool parentIsSubQuery)
protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath,
bool parentIsSubQuery)
{
if (expression.Left is BinaryExpression && expression.Right is ConstantExpression)
{
Expand Down
Loading

0 comments on commit 664d942

Please sign in to comment.