Bugfix/data migration

common/ASC.Migration/Core/AbstractMigration.cs

@@ -54,7 +54,7 @@ protected void ReportProgress(double value, string status)
 
     public MigrationApiInfo ApiInfo { get => _migrationInfo?.ToApiInfo(); }
 
-    public abstract Task InitAsync(string path, CancellationToken cancellationToken, string operation);
+    public abstract Task InitAsync(string path, CancellationToken cancellationToken, OperationType operation);
 
     public abstract Task<MigrationApiInfo> ParseAsync(bool reportProgress = true);
 

common/ASC.Migration/Core/IMigration.cs

@@ -35,7 +35,7 @@ public interface IMigration : IDisposable
     string GetProgressStatus();
     MigrationApiInfo ApiInfo { get; }
 
-    Task InitAsync(string path, CancellationToken token, string operation);
+    Task InitAsync(string path, CancellationToken token, OperationType type);
 
     Task<MigrationApiInfo> ParseAsync(bool reportProgress);
 

common/ASC.Migration/Core/MigrationOperation.cs

@@ -125,7 +125,7 @@ protected override async Task DoJob()
 
             var discStore = await storageFactory.GetStorageAsync(TenantId, "migration", (IQuotaController)null) as DiscDataStore;
             var folder = discStore.GetPhysicalPath("", "");
-            await migrator.InitAsync(folder, CancellationToken, onlyParse ? "parse" : "migration");
+            await migrator.InitAsync(folder, CancellationToken, onlyParse ? OperationType.Parse : OperationType.Migration);
 
             var result = await migrator.ParseAsync(onlyParse);
             if (!onlyParse)

common/ASC.Migration/Core/Models/MigrationInfo.cs

@@ -38,7 +38,7 @@ public abstract class MigrationInfo<TUser, TFiles, TGroup> : IMigrationInfo
     public Dictionary<string, TUser> ExistUsers = new Dictionary<string, TUser>();
     public string Path { get; set; }
     public string MigratorName { get; set; }
-    public string Operation { get; set; }
+    public OperationType Operation { get; set; }
     public List<string> Files { get; set; }
     public List<string> FailedArchives = new List<string>();
     public List<TGroup> Groups = new List<TGroup>();
@@ -58,7 +58,7 @@ public virtual MigrationApiInfo ToApiInfo()
             SuccessedUsers = SuccessedUsers,
             FailedUsers = FailedUsers,
             Groups = Groups.Select(g => g.ToApiInfo()).ToList(),
-            Operation = Operation,
+            Operation = Operation.ToString().ToLower(),
             Files = Files,
         };
     }

common/ASC.Migration/Core/OperationType.cs

@@ -0,0 +1,32 @@
+// (c) Copyright Ascensio System SIA 2010-2023
+//
+// This program is a free software product.
+// You can redistribute it and/or modify it under the terms
+// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
+// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
+// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
+// any third-party rights.
+//
+// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For details, see
+// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
+//
+// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
+//
+// The  interactive user interfaces in modified source and object code versions of the Program must
+// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
+//
+// Pursuant to Section 7(b) of the License you must retain the original Product logo when
+// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
+// trademark law for use of our trademarks.
+//
+// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
+// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
+// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
+
+namespace ASC.Migration.Core.Core;
+public enum OperationType
+{
+    Parse,
+    Migration
+}

common/ASC.Migration/Core/Providers/ASC/Models/WorkspaceMigratingFiles.cs

@@ -220,6 +220,7 @@ public override async Task MigrateAsync()
                 $"ASC migration files {DateTime.Now:dd.MM.yyyy}")
             : await fileStorageService.CreateRoomAsync($"ASC migration {(_type == FolderType.BUNCH ? "project" : "common")} files {DateTime.Now:dd.MM.yyyy}",
                 RoomType.PublicRoom, false, false, new List<FileShareParams>(), 0);
+        Log($"create root folder", null);
 
         var matchingIds = new Dictionary<string, FileEntry<int>> { { $"{_folderKey}-{_folder}", newFolder } };
 
@@ -230,6 +231,7 @@ public override async Task MigrateAsync()
                 || matchingIds[$"{_folderKey}-{folder.ParentId}"].Id != 0) 
             {
                 newFolder = await fileStorageService.CreateFolderAsync(matchingIds[$"{_folderKey}-{folder.ParentId}"].Id, folder.Title);
+                Log($"create folder {newFolder.Title}", null);
             }
             else
             {
@@ -259,13 +261,10 @@ public override async Task MigrateAsync()
                 {
                     newFile = await fileDao.SaveFileAsync(newFile, fs);
                 }
-                try
+                if (!matchingIds.ContainsKey($"{_fileKey}-{file.Id}"))
                 {
                     matchingIds.Add($"{_fileKey}-{file.Id}", newFile);
-                }
-                catch
-                {
-
+                    Log($"create file {file.Title}", null);
                 }
             }
             catch(Exception ex)
@@ -348,6 +347,7 @@ public override async Task MigrateAsync()
 
                         orderedFolders = _storage.Folders.Where(f => f.ParentId == security.EntryId).OrderBy(f => f.Level);
                         matchingRoomIds.Add(security.EntryId, room);
+                        Log($"create share room {room.Title}", null);
 
                         if (_user.UserType == EmployeeType.Collaborator)
                         {
@@ -375,6 +375,7 @@ public override async Task MigrateAsync()
                         {
                             newFolder = await fileStorageService.CreateFolderAsync(matchingRoomIds[folder.ParentId].Id, folder.Title);
                             matchingRoomIds.Add(folder.Id, newFolder);
+                            Log($"create folder {newFolder.Title}", null);
                         }
                         foreach (var file in _storage.Files.Where(f => matchingRoomIds.ContainsKey(f.Folder)))
                         {
@@ -389,6 +390,7 @@ public override async Task MigrateAsync()
                             newFile.Version = file.Version;
                             newFile.VersionGroup = file.VersionGroup;
                             newFile = await fileDao.SaveFileAsync(newFile, fs);
+                            Log($"create file {newFile.Title}", null);
                         }
                     }
                     if (_user.UserType == EmployeeType.Collaborator && _currentUser.ID == Guid.Parse(_mappedGuids[security.Subject]))

common/ASC.Migration/Core/Providers/ASC/Models/WorkspaceMigrationGroups.cs

@@ -25,8 +25,6 @@
 // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 
 
-using ASC.Migration.Core.Core.Providers.ASC.Models.Parse;
-
 using Constants = ASC.Core.Users.Constants;
 
 namespace ASC.Migration.Core.Core.Providers.ASC.Models;

common/ASC.Migration/Core/Providers/ASC/WorkspaceMigration.cs

@@ -46,7 +46,7 @@ public class WorkspaceMigration(
     private Dictionary<string, string> _mappedGuids;
     public override MigratorMeta Meta => _meta;
 
-    public override async Task InitAsync(string path, CancellationToken cancellationToken, string operation)
+    public override async Task InitAsync(string path, CancellationToken cancellationToken, OperationType operation)
     {
         await _logger.InitAsync();
         _cancellationToken = cancellationToken;
@@ -182,6 +182,7 @@ public override async Task<MigrationApiInfo> ParseAsync(bool reportProgress = tr
         {
             _migrationInfo.FailedArchives.Add(Path.GetFileName(_backup));
             Log($"Couldn't parse {Path.GetFileNameWithoutExtension(_backup)} archive", ex);
+            throw new Exception($"Couldn't parse {Path.GetFileNameWithoutExtension(_backup)} archive");
         }
         if (reportProgress)
         {

common/ASC.Migration/Core/Providers/GoogleWorkspace/GoogleWorkspaceMigration.cs

@@ -24,12 +24,6 @@
 // content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
 // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 
-
-
-using System.Linq;
-
-using net.openstack.Providers.Rackspace.Objects.Databases;
-
 namespace ASC.Migration.GoogleWorkspace;
 
 [Scope]
@@ -46,7 +40,7 @@ public class GoogleWorkspaceMigration(
     private string _path;
     public override MigratorMeta Meta => _meta;
 
-    public override async Task InitAsync(string path, CancellationToken cancellationToken, string operation)
+    public override async Task InitAsync(string path, CancellationToken cancellationToken, OperationType operation)
     {
         await _logger.InitAsync();
         _cancellationToken = cancellationToken;
@@ -140,14 +134,14 @@ public override async Task<MigrationApiInfo> ParseAsync(bool reportProgress = tr
                     }
                     else if ((await userManager.GetUserByEmailAsync(user.Email)) != ASC.Core.Users.Constants.LostUser)
                     {
-                        if (!_migrationInfo.ExistUsers.Any(u => u.Value.Email == user.Email) || _migrationInfo.Operation == "migration")
+                        if (!_migrationInfo.ExistUsers.Any(u => u.Value.Email == user.Email) || _migrationInfo.Operation is OperationType.Migration)
                         {
                             _migrationInfo.ExistUsers.Add(key, user);
                         }
                     }
                     else
                     {
-                        if (!_migrationInfo.Users.Any(u => u.Value.Email == user.Email) || _migrationInfo.Operation == "migration") 
+                        if (!_migrationInfo.Users.Any(u => u.Value.Email == user.Email) || _migrationInfo.Operation is OperationType.Migration) 
                         {
                             _migrationInfo.Users.Add(key, user);
                         }
@@ -158,6 +152,10 @@ public override async Task<MigrationApiInfo> ParseAsync(bool reportProgress = tr
             {
                 _migrationInfo.FailedArchives.Add(key);
                 Log($"Couldn't parse user from {key} archive", ex);
+                if (_migrationInfo.FailedArchives.Count == _takeouts.Length)
+                {
+                    throw new Exception("Couldn't parse arhives");
+                }
             }
             finally
             {
@@ -241,8 +239,6 @@ public override async Task MigrateAsync(MigrationApiInfo migrationApiInfo)
                 continue;
             }
 
-            var smallStep = progressStep / 4;
-
             try
             {
                 var currentUser = securityContext.CurrentAccount;
@@ -259,7 +255,7 @@ public override async Task MigrateAsync(MigrationApiInfo migrationApiInfo)
             }
             finally
             {
-                ReportProgress(GetProgress() + smallStep, string.Format(MigrationResource.MigratingUserFiles, user.DisplayName, i, usersCount));
+                ReportProgress(GetProgress() + progressStep, string.Format(MigrationResource.MigratingUserFiles, user.DisplayName, i, usersCount));
             }
             i++;
         }

common/ASC.Migration/Core/Providers/GoogleWorkspace/Models/GWSMigratingFiles.cs

@@ -149,6 +149,7 @@ public override async Task MigrateAsync()
                 var parentId = await globalFolderHelper.FolderMyAsync;
                 var createdFolder = await fileStorageService.CreateFolderAsync(parentId, _newParentFolder);
                 foldersDict.Add(_newParentFolder, createdFolder);
+                Log($"Сreate root folder", null);
             }
 
             if (_folders != null && _folders.Count != 0)
@@ -231,6 +232,7 @@ public override async Task MigrateAsync()
 
                                     try
                                     {
+                                        Log($"Сreate share room {room.Title}", null);
                                         await fileStorageService.SetAceObjectAsync(aceCollection, false);
                                         await securityContext.AuthenticateMeAsync(_user.Guid);
                                     }
@@ -245,6 +247,7 @@ public override async Task MigrateAsync()
                                 var createdFolder = await fileStorageService.CreateFolderAsync(parentId, split[i]);
                                 path = path.Contains(_newParentFolder + Path.DirectorySeparatorChar.ToString()) ? path.Replace(_newParentFolder + Path.DirectorySeparatorChar.ToString(), "") : path;
                                 foldersDict.Add(path, createdFolder);
+                                Log($"Сreate folder {path}", null);
                             }
                         }
                         catch (Exception ex)
@@ -269,6 +272,7 @@ public override async Task MigrateAsync()
                         var parentFolder = string.IsNullOrWhiteSpace(maskParentPath) ? foldersDict[_newParentFolder] : foldersDict[maskParentPath];
                         var newFile = await AddFileAsync(realPath, parentFolder.Id, Path.GetFileName(file));
                         filesDict.Add(realPath, newFile);
+                        Log($"Сreate file {maskParentPath}/{Path.GetFileName(file)}", null);
                     }
                     catch (Exception ex)
                     {

common/ASC.Migration/Core/Providers/NextcloudWorkspace/NextcloudWorkspaceMigration.cs

@@ -49,7 +49,7 @@ public NextcloudWorkspaceMigration(
         _userManager = userManager;
     }
 
-    public override async Task InitAsync(string path, CancellationToken cancellationToken, string operation)
+    public override async Task InitAsync(string path, CancellationToken cancellationToken, OperationType operation)
     {
         await _logger.InitAsync();
         _cancellationToken = cancellationToken;

common/ASC.Migration/Core/Providers/OwnCloud/OwnCloudMigration.cs

@@ -39,7 +39,7 @@ public class OwnCloudMigration(
     private readonly MigratorMeta _meta = new("Owncloud", 6, false);
     public override MigratorMeta Meta => _meta;
 
-    public override async Task InitAsync(string path, CancellationToken cancellationToken, string operation)
+    public override async Task InitAsync(string path, CancellationToken cancellationToken, OperationType operation)
     {
         await _logger.InitAsync();
         _cancellationToken = cancellationToken;