diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index 3d2fe47d326..220e84c2e0d 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -99,7 +99,6 @@ public async IAsyncEnumerable> IndexAllAsync( Func> getIds, Func> getData) { - var now = DateTime.UtcNow; DateTime lastIndexed; await using (var webStudioDbContext = await _dbContextFactory.CreateDbContextAsync()) @@ -123,13 +122,16 @@ public async IAsyncEnumerable> IndexAllAsync( { yield return getData(ids[i], ids[i + 1], lastIndexed); } + } + public async Task OnComplete(DateTime lastModified) + { await using (var webStudioDbContext = await _dbContextFactory.CreateDbContextAsync()) { await webStudioDbContext.AddOrUpdateAsync(q => q.WebstudioIndex, new DbWebstudioIndex { IndexName = Wrapper.IndexName, - LastModified = now + LastModified = lastModified }); await webStudioDbContext.SaveChangesAsync(); @@ -137,8 +139,8 @@ public async IAsyncEnumerable> IndexAllAsync( _logger.DebugIndexCompleted(Wrapper.IndexName); } - - public async Task ReIndrexAsync() + + public async Task ReIndexAsync() { await ClearAsync(); } diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 7521b795ff8..21fe4874f6c 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -226,7 +226,7 @@ public async Task Index(List data, bool immediately = true, int retry = 0) } else if (e.Response.HttpStatusCode == 429) { - Thread.Sleep(60000); + await Task.Delay(60000); if (retry < 10) { await Index(data.Where(r => r != null).ToList(), immediately, retry + 1); @@ -251,87 +251,6 @@ public async Task Index(List data, bool immediately = true, int retry = 0) { Logger.ErrorInner(inner); - if (inner.Response.HttpStatusCode is 413 or 403) - { - Logger.Error(inner.Response.HttpStatusCode.ToString()); - foreach (var r in data.Where(r => r != null)) - { - await Index(r, immediately); - } - } - else if (inner.Response.HttpStatusCode == 429) - { - Thread.Sleep(60000); - if (retry < 10) - { - await Index(data.Where(r => r != null).ToList(), immediately, retry + 1); - return; - } - - throw; - } - } - else - { - throw; - } - } - } - - protected async Task IndexAsync(List data, bool immediately = true, int retry = 0) - { - var t = _serviceProvider.GetService(); - if (!await SupportAsync(t) || data.Count == 0) - { - return; - } - - try - { - await _indexer.IndexAsync(data, immediately).ConfigureAwait(false); - } - catch (ElasticsearchClientException e) - { - Logger.ErrorIndexAsync(e); - - if (e.Response != null) - { - Logger.Error(e.Response.HttpStatusCode.ToString()); - - if (e.Response.HttpStatusCode is 413 or 403 or 408) - { - foreach (var r in data.Where(r => r != null)) - { - await Index(r, immediately); - } - } - else if (e.Response.HttpStatusCode == 429) - { - await Task.Delay(60000); - if (retry < 10) - { - await IndexAsync(data.Where(r => r != null).ToList(), immediately, retry + 1); - return; - } - - throw; - } - } - } - catch (AggregateException e) //ElasticsearchClientException - { - if (e.InnerExceptions.Count == 0) - { - throw; - } - - var inner = e.InnerExceptions.OfType().FirstOrDefault(); - - - if (inner != null) - { - Logger.ErrorIndexAsync(inner); - if (inner.Response.HttpStatusCode is 413 or 403) { Logger.Error(inner.Response.HttpStatusCode.ToString()); @@ -345,7 +264,7 @@ protected async Task IndexAsync(List data, bool immediately = true, int retry await Task.Delay(60000); if (retry < 10) { - await IndexAsync(data.Where(r => r != null).ToList(), immediately, retry + 1); + await Index(data.Where(r => r != null).ToList(), immediately, retry + 1); return; } @@ -533,7 +452,7 @@ public virtual Task IndexAllAsync() public async Task ReIndexAsync() { - await _indexer.ReIndrexAsync(); + await _indexer.ReIndexAsync(); } public bool Support(T t) diff --git a/common/services/ASC.ElasticSearch/Log/FactoryIndexerLogger.cs b/common/services/ASC.ElasticSearch/Log/FactoryIndexerLogger.cs index 983a2b00b3e..96a7c2cd418 100644 --- a/common/services/ASC.ElasticSearch/Log/FactoryIndexerLogger.cs +++ b/common/services/ASC.ElasticSearch/Log/FactoryIndexerLogger.cs @@ -42,9 +42,6 @@ internal static partial class FactoryIndexerLogger [LoggerMessage(Level = LogLevel.Error, Message = "inner")] public static partial void ErrorInner(this ILogger logger, Exception exception); - [LoggerMessage(Level = LogLevel.Error, Message = "IndexAsync")] - public static partial void ErrorIndexAsync(this ILogger logger, Exception exception); - [LoggerMessage(Level = LogLevel.Error, Message = "Update")] public static partial void ErrorUpdate(this ILogger logger, Exception exception); diff --git a/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs b/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs index c1f1f42778f..fbb08084763 100644 --- a/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs +++ b/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs @@ -94,7 +94,8 @@ public override async Task IndexAllAsync() { var j = 0; var tasks = new List(); - + var now = DateTime.UtcNow; + await foreach (var data in _indexer.IndexAllAsync(GetCount, GetIds, GetData)) { if (_settings.Threads == 1) @@ -103,7 +104,7 @@ public override async Task IndexAllAsync() } else { - tasks.Add(IndexAsync(data)); + tasks.Add(Index(data)); j++; if (j >= _settings.Threads) { @@ -118,6 +119,8 @@ public override async Task IndexAllAsync() { Task.WaitAll(tasks.ToArray()); } + + await _indexer.OnComplete(now); } catch (Exception e) { diff --git a/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs b/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs index 5733dd1cc3a..3baa60855b1 100644 --- a/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs +++ b/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs @@ -50,26 +50,47 @@ public FactoryIndexerFolder( public override async Task IndexAllAsync() { - (int, int, int) getCount(DateTime lastIndexed) + try { - using var filesDbContext = _dbContextFactory.CreateDbContext(); - - var minid = Queries.FolderMinId(filesDbContext, lastIndexed); - - var maxid = Queries.FolderMaxId(filesDbContext, lastIndexed); - - var count = Queries.FoldersCount(filesDbContext, lastIndexed); + var j = 0; + var tasks = new List(); + var now = DateTime.UtcNow; + + await foreach (var data in _indexer.IndexAllAsync(GetCount, GetIds, GetData)) + { + if (_settings.Threads == 1) + { + await Index(data); + } + else + { + tasks.Add(Index(data)); + j++; + if (j >= _settings.Threads) + { + Task.WaitAll(tasks.ToArray()); + tasks = new List(); + j = 0; + } + } + } - return new(count, maxid, minid); + if (tasks.Count > 0) + { + Task.WaitAll(tasks.ToArray()); + } + + await _indexer.OnComplete(now); } - - List getData(long start, long stop, DateTime lastIndexed) + catch (Exception e) { - using var filesDbContext = _dbContextFactory.CreateDbContext(); - return Queries.FolderData(filesDbContext, lastIndexed, start, stop).ToList(); + Logger.ErrorFactoryIndexerFolder(e); + throw; } - List getIds(DateTime lastIndexed) + return; + + List GetIds(DateTime lastIndexed) { var start = 0; var result = new List(); @@ -93,39 +114,23 @@ List getIds(DateTime lastIndexed) return result; } - try + List GetData(long start, long stop, DateTime lastIndexed) { - var j = 0; - var tasks = new List(); - - await foreach (var data in _indexer.IndexAllAsync(getCount, getIds, getData)) - { - if (_settings.Threads == 1) - { - await Index(data); - } - else - { - tasks.Add(IndexAsync(data)); - j++; - if (j >= _settings.Threads) - { - Task.WaitAll(tasks.ToArray()); - tasks = new List(); - j = 0; - } - } - } - - if (tasks.Count > 0) - { - Task.WaitAll(tasks.ToArray()); - } + using var filesDbContext = _dbContextFactory.CreateDbContext(); + return Queries.FolderData(filesDbContext, lastIndexed, start, stop).ToList(); } - catch (Exception e) + + (int, int, int) GetCount(DateTime lastIndexed) { - Logger.ErrorFactoryIndexerFolder(e); - throw; + using var filesDbContext = _dbContextFactory.CreateDbContext(); + + var minId = Queries.FolderMinId(filesDbContext, lastIndexed); + + var maxId = Queries.FolderMaxId(filesDbContext, lastIndexed); + + var count = Queries.FoldersCount(filesDbContext, lastIndexed); + + return new(count, maxId, minId); } } }