From 3d99fa618cdf11c1cafac2bbb315f0ac40885aff Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 4 Oct 2021 21:12:30 +0200 Subject: [PATCH] Fix sync of asset folders. --- .../Squidex.CLI.Tests/FolderTreeTests.cs | 280 ++++++++++++++++++ .../Squidex.CLI.Tests.csproj | 2 + .../Implementation/Sync/Assets/FolderNode.cs | 6 +- .../Implementation/Sync/Assets/FolderTree.cs | 81 +++-- .../Sync/Assets/UploadPipeline.cs | 13 +- .../Squidex.CLI/Squidex.CLI.csproj | 2 +- 6 files changed, 354 insertions(+), 30 deletions(-) create mode 100644 cli/Squidex.CLI/Squidex.CLI.Tests/FolderTreeTests.cs diff --git a/cli/Squidex.CLI/Squidex.CLI.Tests/FolderTreeTests.cs b/cli/Squidex.CLI/Squidex.CLI.Tests/FolderTreeTests.cs new file mode 100644 index 00000000..ab1d38ad --- /dev/null +++ b/cli/Squidex.CLI/Squidex.CLI.Tests/FolderTreeTests.cs @@ -0,0 +1,280 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using FakeItEasy; +using Squidex.CLI.Commands.Implementation; +using Squidex.CLI.Commands.Implementation.Sync.Assets; +using Squidex.ClientLibrary.Management; +using Xunit; + +namespace Squidex.CLI.Tests +{ + public class FolderTreeTests + { + private static readonly string RootId = Guid.Empty.ToString(); + private readonly ISession session = A.Fake(); + private readonly IAssetsClient assets = A.Fake(); + private readonly FolderTree sut; + + public FolderTreeTests() + { + A.CallTo(() => session.Assets) + .Returns(assets); + + A.CallTo(() => session.App) + .Returns("my-app"); + + sut = new FolderTree(session); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("/")] + [InlineData(".")] + [InlineData("./")] + [InlineData("\\")] + public async Task Should_provide_null_if_for_root_path(string path) + { + var id = await sut.GetIdAsync(path); + + Assert.Null(id); + } + + [Fact] + public async Task Should_query_for_path_once_for_each_subtree() + { + // * folder1 + // * folder2 + var folder1 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder1" + }; + + var folder2 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder2" + }; + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", folder1.Id, A._)) + .Returns(new AssetFoldersDto + { + Items = new List(), + Path = new List + { + folder1 + } + }); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", folder2.Id, A._)) + .Returns(new AssetFoldersDto + { + Items = new List(), + Path = new List + { + folder2 + } + }); + + Assert.Equal("folder1", await sut.GetPathAsync(folder1.Id)); + Assert.Equal("folder2", await sut.GetPathAsync(folder2.Id)); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", A._, A._)) + .MustHaveHappenedTwiceExactly(); + } + + [Fact] + public async Task Should_not_query_for_path_again_if_child_already_queried() + { + // * folder1 + // * folder2 + var folder1 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder1" + }; + + var folder2 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder2" + }; + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", folder2.Id, A._)) + .Returns(new AssetFoldersDto + { + Items = new List(), + Path = new List + { + folder1, + folder2 + } + }); + + Assert.Equal("folder1/folder2", await sut.GetPathAsync(folder2.Id)); + Assert.Equal("folder1", await sut.GetPathAsync(folder1.Id)); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", A._, A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_not_query_for_path_again_if_parent_already_queried() + { + // * folder1 + // * folder2 + var folder1 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder1" + }; + + var folder2 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder2" + }; + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", folder1.Id, A._)) + .Returns(new AssetFoldersDto + { + Items = new List + { + folder2 + }, + Path = new List + { + folder1 + } + }); + + Assert.Equal("folder1", await sut.GetPathAsync(folder1.Id)); + Assert.Equal("folder1/folder2", await sut.GetPathAsync(folder2.Id)); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", A._, A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_query_for_id_once_for_each_tree_item() + { + // * folder1 + // * folder2 + var folder1 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder1" + }; + + var folder2 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder2" + }; + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", RootId, A._)) + .Returns(new AssetFoldersDto + { + Items = new List + { + folder1, + folder2 + }, + Path = new List() + }); + + Assert.Equal(folder1.Id, await sut.GetIdAsync("folder1")); + Assert.Equal(folder2.Id, await sut.GetIdAsync("folder2")); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", A._, A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_not_query_for_id_again_if_parent_already_queried() + { + // * folder1 + // * folder2 + var folder1 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder1" + }; + + var folder2 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder2" + }; + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", folder1.Id, A._)) + .Returns(new AssetFoldersDto + { + Items = new List + { + folder2 + }, + Path = new List + { + folder1 + } + }); + + Assert.Equal("folder1", await sut.GetPathAsync(folder1.Id)); + Assert.Equal("folder1/folder2", await sut.GetPathAsync(folder2.Id)); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", A._, A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_query_for_id_once_and_create_new_folder() + { + // * folder1 + // * folder2 + var folder1 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder1" + }; + + var folder2 = new AssetFolderDto + { + Id = Guid.NewGuid().ToString(), + FolderName = "folder2" + }; + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", RootId, A._)) + .Returns(new AssetFoldersDto + { + Items = new List + { + folder1 + }, + Path = new List() + }); + + A.CallTo(() => assets.PostAssetFolderAsync("my-app", + A.That.Matches(x => x.FolderName == "folder2" && x.ParentId == RootId), + A._)) + .Returns(folder2); + + Assert.Equal(folder1.Id, await sut.GetIdAsync("folder1")); + Assert.Equal(folder2.Id, await sut.GetIdAsync("folder2")); + + A.CallTo(() => assets.GetAssetFoldersAsync("my-app", A._, A._)) + .MustHaveHappenedOnceExactly(); + } + } +} diff --git a/cli/Squidex.CLI/Squidex.CLI.Tests/Squidex.CLI.Tests.csproj b/cli/Squidex.CLI/Squidex.CLI.Tests/Squidex.CLI.Tests.csproj index 09a26b39..d6fd316b 100644 --- a/cli/Squidex.CLI/Squidex.CLI.Tests/Squidex.CLI.Tests.csproj +++ b/cli/Squidex.CLI/Squidex.CLI.Tests/Squidex.CLI.Tests.csproj @@ -3,8 +3,10 @@ net5.0 + + diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderNode.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderNode.cs index 05113219..aa604945 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderNode.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderNode.cs @@ -17,12 +17,12 @@ public sealed record FolderNode(string Id, string Path) public FolderNode Parent { get; set; } - public FolderNode Add(FolderNode child, string name) + public bool HasBeenQueried { get; set; } + + public void Add(FolderNode child, string name) { Children[name] = child; child.Parent = this; - - return child; } } } diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderTree.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderTree.cs index 55e590d8..11cff0a0 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderTree.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/FolderTree.cs @@ -23,6 +23,8 @@ public sealed class FolderTree public FolderTree(ISession session) { + nodes[RootId] = rootNode; + this.session = session; } @@ -38,21 +40,9 @@ public async Task GetPathAsync(string id) return current.Path; } - var folders = await session.Assets.GetAssetFoldersAsync(session.App, id); - - current = rootNode; - - foreach (var folder in folders.Path) - { - current = TryAdd(current, folder.Id, folder.FolderName); - } - - foreach (var child in folders.Items) - { - TryAdd(current, child.Id, child.FolderName); - } + var folder = await QueryAsync(id, false); - return current.Path; + return folder.Path; } public async Task GetIdAsync(string path) @@ -81,21 +71,63 @@ public async Task GetIdAsync(string path) continue; } - var request = new CreateAssetFolderDto - { - FolderName = name, - ParentId = current.Id - }; + var node = await QueryAsync(current.Id, true); - var folder = await session.Assets.PostAssetFolderAsync(session.App, request); + if (node.Children.TryGetValue(name, out child)) + { + current = child; + continue; + } - current = TryAdd(current, folder.Id, name); + current = await AddFolderAsync(current, name); } return current.Id; } - private static FolderNode TryAdd(FolderNode node, string id, string name) + private async Task AddFolderAsync(FolderNode current, string name) + { + var request = new CreateAssetFolderDto + { + FolderName = name, + ParentId = current.Id + }; + + var folder = await session.Assets.PostAssetFolderAsync(session.App, request); + + current = TryAdd(current, folder.Id, name); + current.HasBeenQueried = true; + + return current; + } + + private async Task QueryAsync(string id, bool needsChildren) + { + if (nodes.TryGetValue(id, out var node) && (node.HasBeenQueried || !needsChildren)) + { + return node; + } + + var folders = await session.Assets.GetAssetFoldersAsync(session.App, id); + + var current = rootNode; + + foreach (var folder in folders.Path) + { + current = TryAdd(current, folder.Id, folder.FolderName); + } + + current.HasBeenQueried = true; + + foreach (var child in folders.Items) + { + TryAdd(current, child.Id, child.FolderName); + } + + return current; + } + + private FolderNode TryAdd(FolderNode node, string id, string name) { if (node.Children.TryGetValue(name, out var child)) { @@ -104,7 +136,10 @@ private static FolderNode TryAdd(FolderNode node, string id, string name) child = new FolderNode(id, GetPath(node, name)); - return node.Add(child, name); + nodes[id] = child; + node.Add(child, name); + + return child; } private static string GetPath(FolderNode node, string name) diff --git a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs index 93374eb0..018213c6 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs +++ b/cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs @@ -34,10 +34,17 @@ public UploadPipeline(ISession session, ILogger log, IFileSystem fs) { var file = new FileParameter(stream, asset.FileName, asset.MimeType); - await session.Assets.PostUpsertAssetAsync(session.App, asset.Id, null, true, file); - } + var result = await session.Assets.PostUpsertAssetAsync(session.App, asset.Id, null, true, file); - log.ProcessCompleted(process); + if (string.Equals(asset.FileHash, result.FileHash, StringComparison.Ordinal)) + { + log.ProcessSkipped(process, "Same hash."); + } + else + { + log.ProcessCompleted(process); + } + } } catch (Exception ex) { diff --git a/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj b/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj index fac904cc..47276319 100644 --- a/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj +++ b/cli/Squidex.CLI/Squidex.CLI/Squidex.CLI.csproj @@ -14,7 +14,7 @@ net5.0 true sq - 7.13 + 7.14