From 46f8698496c59d63be0665d7e579b243affb9501 Mon Sep 17 00:00:00 2001 From: Sukhorukov Anton Date: Thu, 25 Jan 2024 16:29:35 +0300 Subject: [PATCH 1/2] fix bug 66001 --- .../Tasks/BackupPortalTask.cs | 20 ------------------- .../Tasks/RestorePortalTask.cs | 15 -------------- 2 files changed, 35 deletions(-) diff --git a/common/ASC.Data.Backup.Core/Tasks/BackupPortalTask.cs b/common/ASC.Data.Backup.Core/Tasks/BackupPortalTask.cs index c8679bcaa14..75918139139 100644 --- a/common/ASC.Data.Backup.Core/Tasks/BackupPortalTask.cs +++ b/common/ASC.Data.Backup.Core/Tasks/BackupPortalTask.cs @@ -123,26 +123,6 @@ private async Task DoDump(IDataWriteOperator writer) { var databases = new Dictionary, List>(); - try - { - await using var connection = DbFactory.OpenConnection(); - var command = connection.CreateCommand(); - command.CommandText = "select id, connection_string from mail_server_server"; - ExecuteList(command).ForEach(r => - { - var connectionString = GetConnectionString((int)r[0], JsonConvert.DeserializeObject>(Convert.ToString(r[1]))["DbConnection"].ToString()); - - var command = connection.CreateCommand(); - command.CommandText = "show tables"; - var tables = ExecuteList(command).Select(r => Convert.ToString(r[0])).ToList(); - databases.Add(new Tuple(connectionString.Name, connectionString.ConnectionString), tables); - }); - } - catch (Exception e) - { - _logger.ErrorWithException(e); - } - await using (var connection = DbFactory.OpenConnection()) { var command = connection.CreateCommand(); diff --git a/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs b/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs index 2b0b38d8d7b..e694afc04f8 100644 --- a/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs +++ b/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs @@ -213,21 +213,6 @@ private async Task RestoreFromDump(IDataReadOperator dataReader) Task.WaitAll(tasks.ToArray()); } - try - { - await using var connection = DbFactory.OpenConnection(); - var command = connection.CreateCommand(); - command.CommandText = "select id, connection_string from mail_server_server"; - ExecuteList(command).ForEach(r => - { - var connectionString = GetConnectionString((int)r[0], JsonConvert.DeserializeObject>(Convert.ToString(r[1]))["DbConnection"].ToString()); - databases.Add(new Tuple(connectionString.Name, connectionString.ConnectionString), databasesFromDirs[connectionString.Name]); - }); - } - catch (Exception e) - { - _logger.ErrorWithException(e); - } foreach (var database in databases) { From 5b1146292d117fcbc294a6b4583783ceb61d3b5e Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 25 Jan 2024 16:50:20 +0300 Subject: [PATCH 2/2] Search: fix memory --- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 39 +++++++++++-------- .../Core/Core/Dao/TeamlabDao/FileDao.cs | 19 ++++----- .../Core/Core/Search/FactoryIndexerFile.cs | 4 +- .../Core/Core/Search/FactoryIndexerFolder.cs | 2 +- 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index c0b14906029..3d2fe47d326 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -94,14 +94,18 @@ public BaseIndexer( _serviceProvider = serviceProvider; } - public async Task>> IndexAllAsync( + public async IAsyncEnumerable> IndexAllAsync( Func getCount, Func> getIds, Func> getData) - { - await using var webstudioDbContext = await _dbContextFactory.CreateDbContextAsync(); + { var now = DateTime.UtcNow; - var lastIndexed = await Queries.LastIndexedAsync(webstudioDbContext, Wrapper.IndexName); + DateTime lastIndexed; + + await using (var webStudioDbContext = await _dbContextFactory.CreateDbContextAsync()) + { + lastIndexed = await Queries.LastIndexedAsync(webStudioDbContext, Wrapper.IndexName); + } if (lastIndexed.Equals(DateTime.MinValue)) { @@ -111,26 +115,27 @@ public async Task>> IndexAllAsync( var (count, max, min) = getCount(lastIndexed); _logger.DebugIndex(IndexName, count, max, min); - var ids = new List() { min }; + var ids = new List { min }; ids.AddRange(getIds(lastIndexed)); ids.Add(max); + + for (var i = 0; i < ids.Count - 1; i++) + { + yield return getData(ids[i], ids[i + 1], lastIndexed); + } - await webstudioDbContext.AddOrUpdateAsync(q => q.WebstudioIndex, new DbWebstudioIndex() + await using (var webStudioDbContext = await _dbContextFactory.CreateDbContextAsync()) { - IndexName = Wrapper.IndexName, - LastModified = now - }); + await webStudioDbContext.AddOrUpdateAsync(q => q.WebstudioIndex, new DbWebstudioIndex + { + IndexName = Wrapper.IndexName, + LastModified = now + }); - await webstudioDbContext.SaveChangesAsync(); + await webStudioDbContext.SaveChangesAsync(); + } _logger.DebugIndexCompleted(Wrapper.IndexName); - - var list = new List>(); - for (var i = 0; i < ids.Count - 1; i++) - { - list.Add(getData(ids[i], ids[i + 1], lastIndexed)); - } - return list; } public async Task ReIndrexAsync() diff --git a/products/ASC.Files/Core/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Core/Core/Dao/TeamlabDao/FileDao.cs index a3ead1a7bf6..c34f93e8f63 100644 --- a/products/ASC.Files/Core/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Core/Core/Dao/TeamlabDao/FileDao.cs @@ -1575,22 +1575,17 @@ select f } protected internal async Task InitDocumentAsync(DbFile dbFile) - { + { + dbFile.Document = new Document + { + Data = Convert.ToBase64String(Encoding.UTF8.GetBytes("")) + }; + if (!await _factoryIndexer.CanIndexByContentAsync(dbFile)) { - dbFile.Document = new Document - { - Data = Convert.ToBase64String(Encoding.UTF8.GetBytes("")) - }; - return dbFile; } - - return await InternalInitDocumentAsync(dbFile); - } - - private async Task InternalInitDocumentAsync(DbFile dbFile) - { + var file = _serviceProvider.GetService>(); file.Id = dbFile.Id; file.Title = dbFile.Title; diff --git a/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs b/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs index d32a2ebe275..c1f1f42778f 100644 --- a/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs +++ b/products/ASC.Files/Core/Core/Search/FactoryIndexerFile.cs @@ -95,7 +95,7 @@ public override async Task IndexAllAsync() var j = 0; var tasks = new List(); - foreach (var data in await _indexer.IndexAllAsync(GetCount, GetIds, GetData)) + await foreach (var data in _indexer.IndexAllAsync(GetCount, GetIds, GetData)) { if (_settings.Threads == 1) { @@ -125,6 +125,8 @@ public override async Task IndexAllAsync() throw; } + return; + List GetIds(DateTime lastIndexed) { var start = 0; diff --git a/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs b/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs index 9ebb8b02187..5733dd1cc3a 100644 --- a/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs +++ b/products/ASC.Files/Core/Core/Search/FactoryIndexerFolder.cs @@ -98,7 +98,7 @@ List getIds(DateTime lastIndexed) var j = 0; var tasks = new List(); - foreach (var data in await _indexer.IndexAllAsync(getCount, getIds, getData)) + await foreach (var data in _indexer.IndexAllAsync(getCount, getIds, getData)) { if (_settings.Threads == 1) {