From 22b57e0d27b0af063281ce10f5435450e5e26569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Mon, 21 Oct 2024 19:53:12 +0200 Subject: [PATCH 01/15] feat: dynamic object storage --- .../ArmoniK.Core.Adapters.LocalStorage.csproj | 12 +- ...rviceCollectionExt.cs => ObjectBuilder.cs} | 28 +-- Adaptors/LocalStorage/src/ObjectStorage.cs | 12 +- Adaptors/Memory/src/ResultTable.cs | 1 + .../src/Object/ObjectDataModelMapping.cs | 85 -------- Adaptors/MongoDB/src/ObjectStorage.cs | 199 ------------------ Adaptors/MongoDB/src/Options/MongoDB.cs | 4 - Adaptors/MongoDB/src/Options/ObjectStorage.cs | 28 --- Adaptors/MongoDB/src/Options/QueueStorage.cs | 34 --- Adaptors/MongoDB/src/ResultTable.cs | 1 + Adaptors/MongoDB/src/ServiceCollectionExt.cs | 8 - Adaptors/MongoDB/src/TaskTable.cs | 1 + Adaptors/MongoDB/tests/InjectionTests.cs | 39 ---- .../MongoDB/tests/MongoDatabaseProvider.cs | 7 - Adaptors/MongoDB/tests/ObjectStorageTests.cs | 46 ---- .../src/ArmoniK.Core.Adapters.Redis.csproj | 53 ++++- Adaptors/Redis/src/ObjectBuilder.cs | 118 +++++++++++ Adaptors/Redis/src/ObjectStorage.cs | 14 +- Adaptors/Redis/src/ServiceCollectionExt.cs | 124 ----------- Adaptors/Redis/tests/ObjectStorageTests.cs | 3 - .../S3/src/ArmoniK.Core.Adapters.S3.csproj | 46 +++- Adaptors/S3/src/ObjectBuilder.cs | 78 +++++++ Adaptors/S3/src/ObjectStorage.cs | 31 ++- Adaptors/S3/src/ServiceCollectionExt.cs | 84 -------- .../ArmoniK.Core.Adapters.S3.Tests.csproj | 1 - Adaptors/S3/tests/ObjectStorageTests.cs | 24 ++- .../src/Exceptions/ArmoniKException.cs | 2 +- .../Exceptions/ObjectDataNotFoundException.cs | 2 +- .../Storage => Base/src}/IObjectStorage.cs | 11 +- .../src/Auth/Authentication/Authenticator.cs | 2 +- .../InvalidSessionTransitionException.cs | 2 + .../Exceptions/PartitionNotFoundException.cs | 2 + .../src/Exceptions/ResultNotFoundException.cs | 2 + .../Exceptions/SessionNotFoundException.cs | 2 + .../Exceptions/SubmissionClosedException.cs | 2 + .../Exceptions/TaskAlreadyExistsException.cs | 2 + .../TaskAlreadyInFinalStateException.cs | 2 + .../src/Exceptions/TaskCanceledException.cs | 2 + .../src/Exceptions/TaskNotFoundException.cs | 2 + Common/src/Exceptions/TaskPausedException.cs | 2 + Common/src/Exceptions/TimeoutException.cs | 2 + Common/src/Injection/Options/Components.cs | 2 +- Common/src/Injection/ServiceCollectionExt.cs | 32 +-- Common/src/Pollster/Agent.cs | 2 +- Common/src/Pollster/DataPrefetcher.cs | 2 +- Common/src/Pollster/TaskHandler.cs | 1 + Common/src/Storage/TaskLifeCycleHelper.cs | 1 + Common/src/Storage/TaskTableExtensions.cs | 1 + .../src/Stream/Worker/WorkerStreamHandler.cs | 2 +- Common/src/gRPC/Convertors/TaskTableExt.cs | 2 +- Common/src/gRPC/RpcExt.cs | 4 +- Common/src/gRPC/Services/GrpcAuthService.cs | 2 +- .../src/gRPC/Services/GrpcResultsService.cs | 1 + .../src/gRPC/Services/GrpcSessionsService.cs | 1 + .../src/gRPC/Services/GrpcSubmitterService.cs | 1 + Common/src/gRPC/Services/GrpcTasksService.cs | 1 + Common/src/gRPC/Services/Submitter.cs | 2 +- .../AdapterLoading/AdapterLoadingTest.cs | 5 +- Common/tests/Helpers/SimpleObjectStorage.cs | 7 - Common/tests/Helpers/TestDatabaseProvider.cs | 4 - .../tests/Helpers/TestPollingAgentProvider.cs | 4 - Common/tests/Helpers/TestPollsterProvider.cs | 4 - .../tests/Helpers/TestTaskHandlerProvider.cs | 4 - Common/tests/Pollster/PollsterTest.cs | 2 +- Common/tests/Pollster/TaskHandlerTest.cs | 4 +- .../Submitter/GrpcSubmitterServiceTests.cs | 1 + .../IntegrationGrpcSubmitterServiceTest.cs | 1 + Common/tests/Submitter/SubmitterTests.cs | 4 - .../tests/TestBase/ObjectStorageTestBase.cs | 2 +- Common/tests/TestBase/ResultTableTestBase.cs | 1 + Common/tests/TestBase/TaskTableTestBase.cs | 1 + .../ArmoniK.Core.Compute.PollingAgent.csproj | 3 - Compute/PollingAgent/src/Program.cs | 18 +- .../src/ArmoniK.Core.Control.Metrics.csproj | 3 - ...moniK.Core.Control.PartitionMetrics.csproj | 3 - .../src/ArmoniK.Core.Control.Submitter.csproj | 3 - Control/Submitter/src/Program.cs | 18 +- Dockerfile | 24 +++ .../modules/storage/object/local/output.tf | 5 +- .../modules/storage/object/minio/outputs.tf | 13 +- .../modules/storage/object/redis/outputs.tf | 5 +- 81 files changed, 470 insertions(+), 846 deletions(-) rename Adaptors/LocalStorage/src/{ServiceCollectionExt.cs => ObjectBuilder.cs} (69%) delete mode 100644 Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs delete mode 100644 Adaptors/MongoDB/src/ObjectStorage.cs delete mode 100644 Adaptors/MongoDB/src/Options/ObjectStorage.cs delete mode 100644 Adaptors/MongoDB/src/Options/QueueStorage.cs delete mode 100644 Adaptors/MongoDB/tests/ObjectStorageTests.cs create mode 100644 Adaptors/Redis/src/ObjectBuilder.cs delete mode 100644 Adaptors/Redis/src/ServiceCollectionExt.cs create mode 100644 Adaptors/S3/src/ObjectBuilder.cs delete mode 100644 Adaptors/S3/src/ServiceCollectionExt.cs rename {Common => Base}/src/Exceptions/ArmoniKException.cs (97%) rename {Common => Base}/src/Exceptions/ObjectDataNotFoundException.cs (96%) rename {Common/src/Storage => Base/src}/IObjectStorage.cs (88%) diff --git a/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj b/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj index 9c88183b8..f369c28fe 100644 --- a/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj +++ b/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj @@ -1,4 +1,4 @@ - + net8.0 win-x64;linux-x64;linux-arm64 @@ -10,10 +10,18 @@ True true enable + true - + + false + runtime + + + false + runtime + diff --git a/Adaptors/LocalStorage/src/ServiceCollectionExt.cs b/Adaptors/LocalStorage/src/ObjectBuilder.cs similarity index 69% rename from Adaptors/LocalStorage/src/ServiceCollectionExt.cs rename to Adaptors/LocalStorage/src/ObjectBuilder.cs index fb69ba594..1a71d92e2 100644 --- a/Adaptors/LocalStorage/src/ServiceCollectionExt.cs +++ b/Adaptors/LocalStorage/src/ObjectBuilder.cs @@ -15,8 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Utils; @@ -28,34 +27,27 @@ namespace ArmoniK.Core.Adapters.LocalStorage; -public static class ServiceCollectionExt +/// +/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable { + /// [PublicAPI] - public static IServiceCollection AddLocalStorage(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] != "ArmoniK.Adapters.LocalStorage.ObjectStorage") - { - return serviceCollection; - } - serviceCollection.AddOption(configuration, Options.LocalStorage.SettingSection, out Options.LocalStorage storageOptions); - using var _ = logger.BeginNamedScope("Object Local configuration", - ("Path", storageOptions.Path)); - logger.LogDebug("setup local storage"); serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage), sp => new ObjectStorage(storageOptions.Path, storageOptions.ChunkSize, sp.GetRequiredService>())); - - return serviceCollection; } } diff --git a/Adaptors/LocalStorage/src/ObjectStorage.cs b/Adaptors/LocalStorage/src/ObjectStorage.cs index 43607022c..8673201eb 100644 --- a/Adaptors/LocalStorage/src/ObjectStorage.cs +++ b/Adaptors/LocalStorage/src/ObjectStorage.cs @@ -22,9 +22,8 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -102,7 +101,6 @@ public async Task AddOrUpdateAsync(string var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); // Write to temporary file await using var file = File.Open(filename, @@ -145,8 +143,6 @@ public async IAsyncEnumerable GetValuesAsync(string var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); - if (!File.Exists(filename)) { throw new ObjectDataNotFoundException($"The object {key} has not been found in {path_}"); @@ -198,18 +194,12 @@ await TryDeleteAsync(key, } } - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - public Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) { var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); - File.Delete(filename); return Task.FromResult(true); diff --git a/Adaptors/Memory/src/ResultTable.cs b/Adaptors/Memory/src/ResultTable.cs index 9934fe0e3..dd93aa36d 100644 --- a/Adaptors/Memory/src/ResultTable.cs +++ b/Adaptors/Memory/src/ResultTable.cs @@ -25,6 +25,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; diff --git a/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs b/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs deleted file mode 100644 index fa9560aeb..000000000 --- a/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs +++ /dev/null @@ -1,85 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.Threading.Tasks; - -using ArmoniK.Core.Adapters.MongoDB.Common; - -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; - -namespace ArmoniK.Core.Adapters.MongoDB.Object; - -public class ObjectDataModelMapping : IMongoDataModelMapping -{ - public const string Collection = "Object"; - - [BsonId] - public string Id - => $"{Key}.{ChunkIdx}"; - - [BsonElement] - public string Key { get; set; } = ""; - - [BsonElement] - public byte[] Chunk { get; set; } = Array.Empty(); - - [BsonElement] - public int ChunkIdx { get; set; } - - /// - [BsonIgnore] - public string CollectionName { get; } = Collection; - - /// - public async Task InitializeIndexesAsync(IClientSessionHandle sessionHandle, - IMongoCollection collection, - Options.MongoDB options) - { - var keyIndex = Builders.IndexKeys.Hashed(model => model.Key); - var chunkIdxIndex = Builders.IndexKeys.Hashed(model => model.ChunkIdx); - var iDIndex = Builders.IndexKeys.Hashed(model => model.Id); - - var indexModels = new CreateIndexModel[] - { - new(iDIndex, - new CreateIndexOptions - { - Name = nameof(iDIndex), - }), - new(keyIndex, - new CreateIndexOptions - { - Name = nameof(keyIndex), - }), - new(chunkIdxIndex, - new CreateIndexOptions - { - Name = nameof(chunkIdxIndex), - }), - }; - - await collection.Indexes.CreateManyAsync(sessionHandle, - indexModels) - .ConfigureAwait(false); - } - - public Task ShardCollectionAsync(IClientSessionHandle sessionHandle, - Options.MongoDB options) - => Task.CompletedTask; -} diff --git a/Adaptors/MongoDB/src/ObjectStorage.cs b/Adaptors/MongoDB/src/ObjectStorage.cs deleted file mode 100644 index 0b84fcee9..000000000 --- a/Adaptors/MongoDB/src/ObjectStorage.cs +++ /dev/null @@ -1,199 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Adapters.MongoDB.Common; -using ArmoniK.Core.Adapters.MongoDB.Object; -using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; - -using MongoDB.Driver; -using MongoDB.Driver.Linq; - -namespace ArmoniK.Core.Adapters.MongoDB; - -[PublicAPI] -public class ObjectStorage : IObjectStorage -{ - private readonly ILogger logger_; - private readonly MongoCollectionProvider objectCollectionProvider_; - private readonly string objectStorageName_; - private readonly SessionProvider sessionProvider_; - private bool isInitialized_; - - public ObjectStorage(SessionProvider sessionProvider, - MongoCollectionProvider objectCollectionProvider, - ILogger logger, - Options.ObjectStorage options) - { - if (options.ChunkSize == 0) - { - throw new ArgumentOutOfRangeException(nameof(options), - $"Minimum value for {nameof(Options.ObjectStorage.ChunkSize)} is 1."); - } - - sessionProvider_ = sessionProvider; - objectCollectionProvider_ = objectCollectionProvider; - objectStorageName_ = "storage/"; - ChunkSize = options.ChunkSize; - logger_ = logger; - } - - public int ChunkSize { get; } - - /// - public async Task Init(CancellationToken cancellationToken) - { - if (!isInitialized_) - { - await sessionProvider_.Init(cancellationToken) - .ConfigureAwait(false); - sessionProvider_.Get(); - - await objectCollectionProvider_.Init(cancellationToken) - .ConfigureAwait(false); - objectCollectionProvider_.Get(); - } - - isInitialized_ = true; - } - - /// - public Task Check(HealthCheckTag tag) - => Task.FromResult(isInitialized_ - ? HealthCheckResult.Healthy() - : HealthCheckResult.Unhealthy()); - - /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - { - long size = 0; - var dbKey = objectStorageName_ + key; - using var _ = logger_.LogFunction(dbKey); - var objectCollection = objectCollectionProvider_.Get(); - - var taskList = new List(); - - var idx = 0; - await foreach (var chunk in valueChunks.WithCancellation(cancellationToken) - .ConfigureAwait(false)) - { - size += chunk.Length; - taskList.Add(objectCollection.InsertOneAsync(new ObjectDataModelMapping - { - Chunk = chunk.ToArray(), - ChunkIdx = idx, - Key = dbKey, - }, - cancellationToken: cancellationToken)); - ++idx; - } - - // If there was no chunks, add an empty chunk, just so that it could be found in the future - if (idx == 0) - { - taskList.Add(objectCollection.InsertOneAsync(new ObjectDataModelMapping - { - Chunk = Array.Empty(), - ChunkIdx = idx, - Key = dbKey, - }, - cancellationToken: cancellationToken)); - } - - await taskList.WhenAll() - .ConfigureAwait(false); - - return size; - } - - /// - async IAsyncEnumerable IObjectStorage.GetValuesAsync(string key, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - using var _ = logger_.LogFunction(objectStorageName_ + key); - var sessionHandle = sessionProvider_.Get(); - var objectCollection = objectCollectionProvider_.Get(); - - var throwException = true; - await foreach (var chunk in objectCollection.AsQueryable(sessionHandle) - .Where(odm => odm.Key == objectStorageName_ + key) - .OrderBy(odm => odm.ChunkIdx) - .Select(odm => odm.Chunk) - .ToAsyncEnumerable(cancellationToken) - .ConfigureAwait(false)) - { - throwException = false; - yield return chunk; - } - - if (throwException) - { - throw new ObjectDataNotFoundException($"Result {key} not found"); - } - } - - /// - public async Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - - { - using var _ = logger_.LogFunction(objectStorageName_); - var objectCollection = objectCollectionProvider_.Get(); - - var names = keys.Select(key => objectStorageName_ + key); - - await objectCollection.DeleteManyAsync(odm => names.Contains(odm.Key), - cancellationToken) - .ConfigureAwait(false); - logger_.LogInformation("Deleted data with {resultIds}", - keys); - } - - /// - public async IAsyncEnumerable ListKeysAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) - { - using var _ = logger_.LogFunction(); - var sessionHandle = sessionProvider_.Get(); - var objectCollection = objectCollectionProvider_.Get(); - - await foreach (var key in objectCollection.AsQueryable(sessionHandle) - .Where(odm => odm.ChunkIdx == 0) - .Select(odm => odm.Key) - .ToAsyncEnumerable(cancellationToken) - .ConfigureAwait(false)) - { - yield return key; - } - } -} diff --git a/Adaptors/MongoDB/src/Options/MongoDB.cs b/Adaptors/MongoDB/src/Options/MongoDB.cs index dcbf8dfc7..403112a6d 100644 --- a/Adaptors/MongoDB/src/Options/MongoDB.cs +++ b/Adaptors/MongoDB/src/Options/MongoDB.cs @@ -56,10 +56,6 @@ public class MongoDB public TableStorage TableStorage { get; set; } = new(); - public ObjectStorage ObjectStorage { get; set; } = new(); - - public QueueStorage QueueStorage { get; set; } = new(); - public int MaxConnectionPoolSize { get; set; } = 500; public TimeSpan ServerSelectionTimeout { get; set; } = TimeSpan.FromMinutes(2); diff --git a/Adaptors/MongoDB/src/Options/ObjectStorage.cs b/Adaptors/MongoDB/src/Options/ObjectStorage.cs deleted file mode 100644 index d50a98e53..000000000 --- a/Adaptors/MongoDB/src/Options/ObjectStorage.cs +++ /dev/null @@ -1,28 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using JetBrains.Annotations; - -namespace ArmoniK.Core.Adapters.MongoDB.Options; - -[PublicAPI] -public class ObjectStorage -{ - public const string SettingSection = nameof(MongoDB) + ":" + nameof(ObjectStorage); - - public int ChunkSize { get; set; } = 14500000; -} diff --git a/Adaptors/MongoDB/src/Options/QueueStorage.cs b/Adaptors/MongoDB/src/Options/QueueStorage.cs deleted file mode 100644 index 8ca8cb351..000000000 --- a/Adaptors/MongoDB/src/Options/QueueStorage.cs +++ /dev/null @@ -1,34 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; - -using JetBrains.Annotations; - -namespace ArmoniK.Core.Adapters.MongoDB.Options; - -[PublicAPI] -public class QueueStorage -{ - public const string SettingSection = nameof(MongoDB) + ":" + nameof(QueueStorage); - - public TimeSpan LockRefreshPeriodicity { get; set; } = TimeSpan.FromMinutes(2); - - public TimeSpan PollPeriodicity { get; set; } = TimeSpan.FromSeconds(5); - - public TimeSpan LockRefreshExtension { get; set; } = TimeSpan.FromMinutes(5); -} diff --git a/Adaptors/MongoDB/src/ResultTable.cs b/Adaptors/MongoDB/src/ResultTable.cs index a4de3c194..b0a4f357e 100644 --- a/Adaptors/MongoDB/src/ResultTable.cs +++ b/Adaptors/MongoDB/src/ResultTable.cs @@ -27,6 +27,7 @@ using ArmoniK.Core.Adapters.MongoDB.Common; using ArmoniK.Core.Adapters.MongoDB.Table.DataModel; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Adaptors/MongoDB/src/ServiceCollectionExt.cs b/Adaptors/MongoDB/src/ServiceCollectionExt.cs index 9fa9e90a7..a38fd6e3d 100644 --- a/Adaptors/MongoDB/src/ServiceCollectionExt.cs +++ b/Adaptors/MongoDB/src/ServiceCollectionExt.cs @@ -77,14 +77,6 @@ public static IServiceCollection AddMongoStorages(this IServiceCollection servic .AddSingleton(); } - if (components["ObjectStorage"] == "ArmoniK.Adapters.MongoDB.ObjectStorage") - { - services.AddOption(configuration, - Options.ObjectStorage.SettingSection) - .AddSingleton() - .AddSingleton(); - } - services.AddOption(configuration, Options.MongoDB.SettingSection, out var mongoOptions); diff --git a/Adaptors/MongoDB/src/TaskTable.cs b/Adaptors/MongoDB/src/TaskTable.cs index 9823caf54..2f42590f7 100644 --- a/Adaptors/MongoDB/src/TaskTable.cs +++ b/Adaptors/MongoDB/src/TaskTable.cs @@ -27,6 +27,7 @@ using ArmoniK.Core.Adapters.MongoDB.Options; using ArmoniK.Core.Adapters.MongoDB.Table.DataModel; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Adaptors/MongoDB/tests/InjectionTests.cs b/Adaptors/MongoDB/tests/InjectionTests.cs index d46387388..a55a71130 100644 --- a/Adaptors/MongoDB/tests/InjectionTests.cs +++ b/Adaptors/MongoDB/tests/InjectionTests.cs @@ -20,7 +20,6 @@ using System.Diagnostics; using ArmoniK.Core.Adapters.MongoDB.Options; -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Utils; using Microsoft.Extensions.Configuration; @@ -87,9 +86,6 @@ public void SetUp() { $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.TableStorage)}:PollingDelayMax", "00:00:20" }, - { - $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.ObjectStorage)}:ChunkSize", "100000" - }, }; var logger = NullLogger.Instance; @@ -233,23 +229,6 @@ public void ReadTablePollingMaxDelay() options.PollingDelayMax); } - [Test] - public void ObjectOptionsNotNull() - { - var options = provider_!.GetRequiredService(); - - Assert.NotNull(options); - } - - [Test] - public void ReadObjectChunkSize() - { - var options = provider_!.GetRequiredService(); - - Assert.AreEqual(100000, - options.ChunkSize); - } - [Test] public void BuildTableStorage() { @@ -275,22 +254,4 @@ public void TableStorageHasPollingDelayMax() Assert.AreEqual(TimeSpan.FromSeconds(20), table.PollingDelayMax); } - - [Test] - public void BuildObjectStorage() - { - var objectStorage = provider_!.GetRequiredService(); - - Assert.NotNull(objectStorage); - } - - [Test] - public void ObjectStorageFactoryHasBindingToObjectStorage() - { - var objectStorage = provider_!.GetRequiredService(); - - Assert.NotNull(objectStorage); - Assert.AreEqual(typeof(ObjectStorage), - objectStorage.GetType()); - } } diff --git a/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs b/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs index 3d6232496..7d3e75e38 100644 --- a/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs +++ b/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs @@ -82,9 +82,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet { $"{Components.SettingSection}:{nameof(Components.TableStorage)}", "ArmoniK.Adapters.MongoDB.TableStorage" }, - { - $"{Components.SettingSection}:{nameof(Components.ObjectStorage)}", "ArmoniK.Adapters.MongoDB.ObjectStorage" - }, { $"{Components.SettingSection}:{nameof(Components.AuthenticationStorage)}", "ArmoniK.Adapters.MongoDB.AuthenticationTable" @@ -96,10 +93,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.TableStorage)}:{nameof(Options.MongoDB.TableStorage.PollingDelayMax)}", "00:00:10" }, - { - $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.ObjectStorage)}:{nameof(Options.MongoDB.ObjectStorage.ChunkSize)}", - "140000" - }, }; var configuration = new ConfigurationManager(); diff --git a/Adaptors/MongoDB/tests/ObjectStorageTests.cs b/Adaptors/MongoDB/tests/ObjectStorageTests.cs deleted file mode 100644 index 0ac4b90ec..000000000 --- a/Adaptors/MongoDB/tests/ObjectStorageTests.cs +++ /dev/null @@ -1,46 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Common.Tests.TestBase; - -using Microsoft.Extensions.DependencyInjection; - -using NUnit.Framework; - -namespace ArmoniK.Core.Adapters.MongoDB.Tests; - -[TestFixture] -public class ObjectStorageTests : ObjectStorageTestBase -{ - public override void TearDown() - { - ObjectStorage = null; - tableProvider_?.Dispose(); - RunTests = false; - } - - private MongoDatabaseProvider? tableProvider_; - - protected override void GetObjectStorageInstance() - { - tableProvider_ = new MongoDatabaseProvider(serviceConfigurator: collection => collection.AddSingleton()); - var provider = tableProvider_.GetServiceProvider(); - ObjectStorage = provider.GetRequiredService(); - RunTests = true; - } -} diff --git a/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj b/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj index d729d4bf1..da557fc65 100644 --- a/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj +++ b/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj @@ -1,4 +1,4 @@ - + net8.0 win-x64;linux-x64;linux-arm64 @@ -9,6 +9,19 @@ True true enable + true + + + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg @@ -20,7 +33,43 @@ - + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + diff --git a/Adaptors/Redis/src/ObjectBuilder.cs b/Adaptors/Redis/src/ObjectBuilder.cs new file mode 100644 index 000000000..ef702d91d --- /dev/null +++ b/Adaptors/Redis/src/ObjectBuilder.cs @@ -0,0 +1,118 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.IO; +using System.Security.Cryptography.X509Certificates; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using StackExchange.Redis; + +namespace ArmoniK.Core.Adapters.Redis; + +/// +/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + { + // ReSharper disable once InlineOutVariableDeclaration + Options.Redis redisOptions; + serviceCollection.AddOption(configuration, + Options.Redis.SettingSection, + out redisOptions); + + if (!string.IsNullOrEmpty(redisOptions.CredentialsPath)) + { + configuration.AddJsonFile(redisOptions.CredentialsPath, + false, + false); + + serviceCollection.AddOption(configuration, + Options.Redis.SettingSection, + out redisOptions); + + logger.LogTrace("Loaded Redis credentials from file {path}", + redisOptions.CredentialsPath); + } + + if (!string.IsNullOrEmpty(redisOptions.CaPath)) + { + var localTrustStore = new X509Store(StoreName.Root); + var certificateCollection = new X509Certificate2Collection(); + try + { + certificateCollection.ImportFromPemFile(redisOptions.CaPath); + localTrustStore.Open(OpenFlags.ReadWrite); + localTrustStore.AddRange(certificateCollection); + logger.LogTrace("Imported Redis certificate from file {path}", + redisOptions.CaPath); + } + catch (Exception ex) + { + logger.LogError("Root certificate import failed: {error}", + ex.Message); + throw; + } + finally + { + localTrustStore.Close(); + } + } + + var config = new ConfigurationOptions + { + ClientName = redisOptions.ClientName, + ReconnectRetryPolicy = new ExponentialRetry(10), + Ssl = redisOptions.Ssl, + AbortOnConnectFail = true, + SslHost = redisOptions.SslHost, + Password = redisOptions.Password, + User = redisOptions.User, + }; + config.EndPoints.Add(redisOptions.EndpointUrl); + + if (redisOptions.Timeout > 0) + { + config.ConnectTimeout = redisOptions.Timeout; + } + + logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}", + redisOptions.EndpointUrl, + redisOptions.User); + + serviceCollection.AddSingleton(_ => ConnectionMultiplexer.Connect(config, + TextWriter.Null) + .GetDatabase()); + serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); + } +} diff --git a/Adaptors/Redis/src/ObjectStorage.cs b/Adaptors/Redis/src/ObjectStorage.cs index 9acf36ee0..dec9a91e8 100644 --- a/Adaptors/Redis/src/ObjectStorage.cs +++ b/Adaptors/Redis/src/ObjectStorage.cs @@ -22,9 +22,8 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -91,9 +90,8 @@ public async Task AddOrUpdateAsync(string IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { - long size = 0; - var storageNameKey = objectStorageName_ + key; - using var _ = logger_.LogFunction(storageNameKey); + long size = 0; + var storageNameKey = objectStorageName_ + key; var idx = 0; var taskList = new List(); @@ -119,7 +117,6 @@ await taskList.WhenAll() public async IAsyncEnumerable GetValuesAsync(string key, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - using var _ = logger_.LogFunction(objectStorageName_ + key); var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); @@ -151,14 +148,9 @@ public async Task TryDeleteAsync(IEnumerable keys, cancellationToken)) .ConfigureAwait(false); - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - private async Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) { - using var _ = logger_.LogFunction(objectStorageName_ + key); var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); diff --git a/Adaptors/Redis/src/ServiceCollectionExt.cs b/Adaptors/Redis/src/ServiceCollectionExt.cs deleted file mode 100644 index 79c8ed094..000000000 --- a/Adaptors/Redis/src/ServiceCollectionExt.cs +++ /dev/null @@ -1,124 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.IO; -using System.Security.Cryptography.X509Certificates; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -using StackExchange.Redis; - -namespace ArmoniK.Core.Adapters.Redis; - -public static class ServiceCollectionExt -{ - [PublicAPI] - public static IServiceCollection AddRedis(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) - { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] == "ArmoniK.Adapters.Redis.ObjectStorage") - { - // ReSharper disable once InlineOutVariableDeclaration - Options.Redis redisOptions; - serviceCollection.AddOption(configuration, - Options.Redis.SettingSection, - out redisOptions); - - using var _ = logger.BeginNamedScope("Redis configuration", - ("EndpointUrl", redisOptions.EndpointUrl)); - - if (!string.IsNullOrEmpty(redisOptions.CredentialsPath)) - { - configuration.AddJsonFile(redisOptions.CredentialsPath, - false, - false); - - serviceCollection.AddOption(configuration, - Options.Redis.SettingSection, - out redisOptions); - - logger.LogTrace("Loaded Redis credentials from file {path}", - redisOptions.CredentialsPath); - } - - if (!string.IsNullOrEmpty(redisOptions.CaPath)) - { - var localTrustStore = new X509Store(StoreName.Root); - var certificateCollection = new X509Certificate2Collection(); - try - { - certificateCollection.ImportFromPemFile(redisOptions.CaPath); - localTrustStore.Open(OpenFlags.ReadWrite); - localTrustStore.AddRange(certificateCollection); - logger.LogTrace("Imported Redis certificate from file {path}", - redisOptions.CaPath); - } - catch (Exception ex) - { - logger.LogError("Root certificate import failed: {error}", - ex.Message); - throw; - } - finally - { - localTrustStore.Close(); - } - } - - var config = new ConfigurationOptions - { - ClientName = redisOptions.ClientName, - ReconnectRetryPolicy = new ExponentialRetry(10), - Ssl = redisOptions.Ssl, - AbortOnConnectFail = true, - SslHost = redisOptions.SslHost, - Password = redisOptions.Password, - User = redisOptions.User, - }; - config.EndPoints.Add(redisOptions.EndpointUrl); - - if (redisOptions.Timeout > 0) - { - config.ConnectTimeout = redisOptions.Timeout; - } - - logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}", - redisOptions.EndpointUrl, - redisOptions.User); - - serviceCollection.AddSingleton(_ => ConnectionMultiplexer.Connect(config, - TextWriter.Null) - .GetDatabase()); - serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); - } - - return serviceCollection; - } -} diff --git a/Adaptors/Redis/tests/ObjectStorageTests.cs b/Adaptors/Redis/tests/ObjectStorageTests.cs index a133b4753..db2544f84 100644 --- a/Adaptors/Redis/tests/ObjectStorageTests.cs +++ b/Adaptors/Redis/tests/ObjectStorageTests.cs @@ -53,9 +53,6 @@ protected override void GetObjectStorageInstance() // Minimal set of configurations to operate on a toy DB Dictionary minimalConfig = new() { - { - "Components:ObjectStorage", "ArmoniK.Adapters.Redis.ObjectStorage" - }, { "Redis:MaxRetry", "5" }, diff --git a/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj b/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj index 42ee40a83..92d2d6b87 100644 --- a/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj +++ b/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj @@ -2,8 +2,14 @@ net8.0 win-x64;linux-x64;linux-arm64 - enable + ANEO + Copyright (C) ANEO, 2021-2021 + AGPL-3.0-or-later + True + True + true enable + true @@ -17,7 +23,43 @@ - + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + diff --git a/Adaptors/S3/src/ObjectBuilder.cs b/Adaptors/S3/src/ObjectBuilder.cs new file mode 100644 index 000000000..9432a1709 --- /dev/null +++ b/Adaptors/S3/src/ObjectBuilder.cs @@ -0,0 +1,78 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using Amazon.S3; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.S3; + +/// +/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + { + // ReSharper disable once InlineOutVariableDeclaration + Options.S3 s3Options; + serviceCollection.AddOption(configuration, + Options.S3.SettingSection, + out s3Options); + + logger.LogInformation("setup connection to S3 at {EndpointUrl} with user {user} with option ForcePathStyle = {ForcePathStyle} with BucketName = {BucketName}", + s3Options.EndpointUrl, + s3Options.Login, + s3Options.MustForcePathStyle, + s3Options.BucketName); + + var s3Config = new AmazonS3Config + { + ForcePathStyle = s3Options.MustForcePathStyle, + ServiceURL = s3Options.EndpointUrl, + }; + + AmazonS3Client s3Client; + if (string.IsNullOrWhiteSpace(s3Options.Login)) + { + s3Client = new AmazonS3Client(s3Config); + } + else + { + s3Client = new AmazonS3Client(s3Options.Login, + s3Options.Password, + s3Config); + } + + + serviceCollection.AddSingleton(_ => s3Client); + serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); + } +} diff --git a/Adaptors/S3/src/ObjectStorage.cs b/Adaptors/S3/src/ObjectStorage.cs index d8bb7bf0a..eac8a7015 100644 --- a/Adaptors/S3/src/ObjectStorage.cs +++ b/Adaptors/S3/src/ObjectStorage.cs @@ -15,15 +15,20 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Util; -using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -93,10 +98,8 @@ public Task Check(HealthCheckTag tag) public async IAsyncEnumerable GetValuesAsync(string key, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var objectStorageFullName = $"{objectStorageName_}{key}"; + try { await s3Client_.GetObjectAsync(options_.BucketName, @@ -166,10 +169,6 @@ public async Task TryDeleteAsync(IEnumerable keys, cancellationToken)) .ConfigureAwait(false); - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - /// public async Task AddOrUpdateAsync(string key, IAsyncEnumerable> valueChunks, @@ -180,10 +179,8 @@ public async Task AddOrUpdateAsync(string { 0, }; - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var objectStorageFullName = $"{objectStorageName_}{key}"; + logger_.LogDebug("Upload object"); var initRequest = new InitiateMultipartUploadRequest { @@ -240,10 +237,8 @@ await s3Client_.AbortMultipartUploadAsync(abortMpuRequest, private async Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) { - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var objectStorageFullName = $"{objectStorageName_}{key}"; + var objectDeleteRequest = new DeleteObjectRequest { BucketName = options_.BucketName, diff --git a/Adaptors/S3/src/ServiceCollectionExt.cs b/Adaptors/S3/src/ServiceCollectionExt.cs deleted file mode 100644 index 97d415ba8..000000000 --- a/Adaptors/S3/src/ServiceCollectionExt.cs +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using Amazon.S3; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace ArmoniK.Core.Adapters.S3; - -public static class ServiceCollectionExt -{ - [PublicAPI] - public static IServiceCollection AddS3(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) - { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] == "ArmoniK.Adapters.S3.ObjectStorage") - { - // ReSharper disable once InlineOutVariableDeclaration - Options.S3 s3Options; - serviceCollection.AddOption(configuration, - Options.S3.SettingSection, - out s3Options); - - using var _ = logger.BeginNamedScope("S3 configuration", - ("EndpointUrl", s3Options.EndpointUrl)); - - logger.LogInformation("setup connection to S3 at {EndpointUrl} with user {user} with option ForcePathStyle = {ForcePathStyle} with BucketName = {BucketName}", - s3Options.EndpointUrl, - s3Options.Login, - s3Options.MustForcePathStyle, - s3Options.BucketName); - - var s3Config = new AmazonS3Config - { - ForcePathStyle = s3Options.MustForcePathStyle, - ServiceURL = s3Options.EndpointUrl, - }; - - AmazonS3Client s3Client; - if (string.IsNullOrWhiteSpace(s3Options.Login)) - { - s3Client = new AmazonS3Client(s3Config); - } - else - { - s3Client = new AmazonS3Client(s3Options.Login, - s3Options.Password, - s3Config); - } - - - serviceCollection.AddSingleton(_ => s3Client); - serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); - } - - return serviceCollection; - } -} diff --git a/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj b/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj index 41d99c8ab..ae9d8bc16 100644 --- a/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj +++ b/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj @@ -27,7 +27,6 @@ - diff --git a/Adaptors/S3/tests/ObjectStorageTests.cs b/Adaptors/S3/tests/ObjectStorageTests.cs index 05996f044..663282232 100644 --- a/Adaptors/S3/tests/ObjectStorageTests.cs +++ b/Adaptors/S3/tests/ObjectStorageTests.cs @@ -15,6 +15,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Common.Utils; @@ -35,12 +37,25 @@ public override void TearDown() RunTests = false; } + private static readonly string SolutionRoot = + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests) + .Assembly + .Location))))) ?? + string.Empty)); + + private static readonly string S3Path = + $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.S3.dll"; + + protected override void GetObjectStorageInstance() { Dictionary minimalConfig = new() { { - "Components:ObjectStorage", "ArmoniK.Adapters.S3.ObjectStorage" + "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.S3.ObjectBuilder" + }, + { + "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}" }, { "S3:BucketName", "miniobucket" @@ -66,10 +81,9 @@ protected override void GetObjectStorageInstance() services.AddLogging(); var logger = new LoggerInit(configuration); - services.AddS3(configuration, - logger.GetLogger()); - - services.AddSingleton(); + services.AddAdapter(configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()); var provider = services.BuildServiceProvider(new ServiceProviderOptions { diff --git a/Common/src/Exceptions/ArmoniKException.cs b/Base/src/Exceptions/ArmoniKException.cs similarity index 97% rename from Common/src/Exceptions/ArmoniKException.cs rename to Base/src/Exceptions/ArmoniKException.cs index 1412f565b..339229974 100644 --- a/Common/src/Exceptions/ArmoniKException.cs +++ b/Base/src/Exceptions/ArmoniKException.cs @@ -17,7 +17,7 @@ using System; -namespace ArmoniK.Core.Common.Exceptions; +namespace ArmoniK.Core.Base.Exceptions; /// /// Base exception for exceptions in ArmoniK core diff --git a/Common/src/Exceptions/ObjectDataNotFoundException.cs b/Base/src/Exceptions/ObjectDataNotFoundException.cs similarity index 96% rename from Common/src/Exceptions/ObjectDataNotFoundException.cs rename to Base/src/Exceptions/ObjectDataNotFoundException.cs index bcf9ebe07..7a8c1ba4f 100644 --- a/Common/src/Exceptions/ObjectDataNotFoundException.cs +++ b/Base/src/Exceptions/ObjectDataNotFoundException.cs @@ -17,7 +17,7 @@ using System; -namespace ArmoniK.Core.Common.Exceptions; +namespace ArmoniK.Core.Base.Exceptions; [Serializable] public class ObjectDataNotFoundException : ArmoniKException diff --git a/Common/src/Storage/IObjectStorage.cs b/Base/src/IObjectStorage.cs similarity index 88% rename from Common/src/Storage/IObjectStorage.cs rename to Base/src/IObjectStorage.cs index 87315c2c4..2be7f95c6 100644 --- a/Common/src/Storage/IObjectStorage.cs +++ b/Base/src/IObjectStorage.cs @@ -21,7 +21,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; namespace ArmoniK.Core.Common.Storage; @@ -69,13 +69,4 @@ IAsyncEnumerable GetValuesAsync(string key, /// the key is not found Task TryDeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default); - - /// - /// List data in the Object Storage - /// - /// Token used to cancel the execution of the method - /// - /// The keys representing data found in the Object Storage - /// - IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default); } diff --git a/Common/src/Auth/Authentication/Authenticator.cs b/Common/src/Auth/Authentication/Authenticator.cs index 6a2df81b6..9d76f39ed 100644 --- a/Common/src/Auth/Authentication/Authenticator.cs +++ b/Common/src/Auth/Authentication/Authenticator.cs @@ -22,8 +22,8 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authorization; -using ArmoniK.Core.Common.Exceptions; using JetBrains.Annotations; diff --git a/Common/src/Exceptions/InvalidSessionTransitionException.cs b/Common/src/Exceptions/InvalidSessionTransitionException.cs index 79abb5156..9d7a6c9a5 100644 --- a/Common/src/Exceptions/InvalidSessionTransitionException.cs +++ b/Common/src/Exceptions/InvalidSessionTransitionException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/PartitionNotFoundException.cs b/Common/src/Exceptions/PartitionNotFoundException.cs index 257c8d57a..bace4fc57 100644 --- a/Common/src/Exceptions/PartitionNotFoundException.cs +++ b/Common/src/Exceptions/PartitionNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/ResultNotFoundException.cs b/Common/src/Exceptions/ResultNotFoundException.cs index 74ce230f3..6d76cc1a3 100644 --- a/Common/src/Exceptions/ResultNotFoundException.cs +++ b/Common/src/Exceptions/ResultNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/SessionNotFoundException.cs b/Common/src/Exceptions/SessionNotFoundException.cs index c58e67f09..443f19675 100644 --- a/Common/src/Exceptions/SessionNotFoundException.cs +++ b/Common/src/Exceptions/SessionNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/SubmissionClosedException.cs b/Common/src/Exceptions/SubmissionClosedException.cs index 2894f5e9c..89b2db80a 100644 --- a/Common/src/Exceptions/SubmissionClosedException.cs +++ b/Common/src/Exceptions/SubmissionClosedException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskAlreadyExistsException.cs b/Common/src/Exceptions/TaskAlreadyExistsException.cs index 8dcd080c7..6cf3003a9 100644 --- a/Common/src/Exceptions/TaskAlreadyExistsException.cs +++ b/Common/src/Exceptions/TaskAlreadyExistsException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs b/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs index 0df0b44c6..5c6c98d3a 100644 --- a/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs +++ b/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskCanceledException.cs b/Common/src/Exceptions/TaskCanceledException.cs index eab3eb11b..ba7258efe 100644 --- a/Common/src/Exceptions/TaskCanceledException.cs +++ b/Common/src/Exceptions/TaskCanceledException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskNotFoundException.cs b/Common/src/Exceptions/TaskNotFoundException.cs index fa0851a8a..fe348ba75 100644 --- a/Common/src/Exceptions/TaskNotFoundException.cs +++ b/Common/src/Exceptions/TaskNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskPausedException.cs b/Common/src/Exceptions/TaskPausedException.cs index 474f33049..8e959912f 100644 --- a/Common/src/Exceptions/TaskPausedException.cs +++ b/Common/src/Exceptions/TaskPausedException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TimeoutException.cs b/Common/src/Exceptions/TimeoutException.cs index 06042151a..762f9f365 100644 --- a/Common/src/Exceptions/TimeoutException.cs +++ b/Common/src/Exceptions/TimeoutException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Injection/Options/Components.cs b/Common/src/Injection/Options/Components.cs index 905666df0..835134dc6 100644 --- a/Common/src/Injection/Options/Components.cs +++ b/Common/src/Injection/Options/Components.cs @@ -43,7 +43,7 @@ public class Components /// /// Represents which object storage is used to store data for tasks /// - public string? ObjectStorage { get; set; } + public AdapterSettings ObjectStorageAdaptorSettings { get; set; } = new(); /// /// Represents which database is used for authentication diff --git a/Common/src/Injection/ServiceCollectionExt.cs b/Common/src/Injection/ServiceCollectionExt.cs index ee56fc356..5e7a8c61e 100644 --- a/Common/src/Injection/ServiceCollectionExt.cs +++ b/Common/src/Injection/ServiceCollectionExt.cs @@ -80,35 +80,37 @@ public static IServiceCollection AddArmoniKWorkerConnection(this IServiceCollect return services; } - public static IServiceCollection AddQueue(this IServiceCollection services, - ConfigurationManager configuration, - ILogger logger) + public static IServiceCollection AddAdapter(this IServiceCollection services, + ConfigurationManager configuration, + string storage, + ILogger logger) { - var queueSettings = configuration.GetRequiredValue($"{Components.SettingSection}:{nameof(Components.QueueAdaptorSettings)}"); + var settings = configuration.GetRequiredValue($"{Components.SettingSection}:{storage}"); - logger.LogInformation("Queue settings for loading adapter {@queueSettings}", - queueSettings); + logger.LogInformation("{storage} settings for loading adapter {@settings}", + storage, + settings); logger.LogDebug("{path}", - queueSettings.AdapterAbsolutePath); + settings.AdapterAbsolutePath); logger.LogDebug("{class}", - queueSettings.ClassName); + settings.ClassName); - if (string.IsNullOrEmpty(queueSettings.AdapterAbsolutePath)) + if (string.IsNullOrEmpty(settings.AdapterAbsolutePath)) { - throw new InvalidOperationException($"{nameof(queueSettings.AdapterAbsolutePath)} should not be null or empty."); + throw new InvalidOperationException($"{nameof(settings.AdapterAbsolutePath)} should not be null or empty."); } - if (string.IsNullOrEmpty(queueSettings.ClassName)) + if (string.IsNullOrEmpty(settings.ClassName)) { - throw new InvalidOperationException($"{nameof(queueSettings.ClassName)} should not be null or empty."); + throw new InvalidOperationException($"{nameof(settings.ClassName)} should not be null or empty."); } - var ctx = new AdapterLoadContext(queueSettings.AdapterAbsolutePath); - var assembly = ctx.LoadFromAssemblyName(AssemblyName.GetAssemblyName(queueSettings.AdapterAbsolutePath)); + var ctx = new AdapterLoadContext(settings.AdapterAbsolutePath); + var assembly = ctx.LoadFromAssemblyName(AssemblyName.GetAssemblyName(settings.AdapterAbsolutePath)); logger.LogInformation("Loaded assembly {assemblyName}", assembly.FullName); - var type = assembly.GetType(queueSettings.ClassName, + var type = assembly.GetType(settings.ClassName, true, true); diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index 9a5f476ee..fd95c9724 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -26,7 +26,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Api.gRPC.V1; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Common/src/Pollster/DataPrefetcher.cs b/Common/src/Pollster/DataPrefetcher.cs index cb6ca1f5a..26a17600c 100644 --- a/Common/src/Pollster/DataPrefetcher.cs +++ b/Common/src/Pollster/DataPrefetcher.cs @@ -23,7 +23,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; diff --git a/Common/src/Pollster/TaskHandler.cs b/Common/src/Pollster/TaskHandler.cs index ecb57e541..73daabfc8 100644 --- a/Common/src/Pollster/TaskHandler.cs +++ b/Common/src/Pollster/TaskHandler.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Meter; diff --git a/Common/src/Storage/TaskLifeCycleHelper.cs b/Common/src/Storage/TaskLifeCycleHelper.cs index 484d76879..99c337cb0 100644 --- a/Common/src/Storage/TaskLifeCycleHelper.cs +++ b/Common/src/Storage/TaskLifeCycleHelper.cs @@ -24,6 +24,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Utils; diff --git a/Common/src/Storage/TaskTableExtensions.cs b/Common/src/Storage/TaskTableExtensions.cs index 7a14a88be..e299bb68b 100644 --- a/Common/src/Storage/TaskTableExtensions.cs +++ b/Common/src/Storage/TaskTableExtensions.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using Microsoft.Extensions.Logging; diff --git a/Common/src/Stream/Worker/WorkerStreamHandler.cs b/Common/src/Stream/Worker/WorkerStreamHandler.cs index 115de8541..0a5a209c8 100644 --- a/Common/src/Stream/Worker/WorkerStreamHandler.cs +++ b/Common/src/Stream/Worker/WorkerStreamHandler.cs @@ -23,7 +23,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Worker; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Storage; diff --git a/Common/src/gRPC/Convertors/TaskTableExt.cs b/Common/src/gRPC/Convertors/TaskTableExt.cs index 2f566c744..72bf7c68a 100644 --- a/Common/src/gRPC/Convertors/TaskTableExt.cs +++ b/Common/src/gRPC/Convertors/TaskTableExt.cs @@ -20,7 +20,7 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Submitter; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using TaskStatus = ArmoniK.Core.Common.Storage.TaskStatus; diff --git a/Common/src/gRPC/RpcExt.cs b/Common/src/gRPC/RpcExt.cs index 9885482f8..386be962e 100644 --- a/Common/src/gRPC/RpcExt.cs +++ b/Common/src/gRPC/RpcExt.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -22,7 +22,7 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using Grpc.Core; diff --git a/Common/src/gRPC/Services/GrpcAuthService.cs b/Common/src/gRPC/Services/GrpcAuthService.cs index 78ee74944..9d1f0cb8e 100644 --- a/Common/src/gRPC/Services/GrpcAuthService.cs +++ b/Common/src/gRPC/Services/GrpcAuthService.cs @@ -20,10 +20,10 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Auth; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Auth.Authorization.Permissions; -using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Meter; using Grpc.Core; diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index e3cf424fc..5c2bc96ef 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1.Results; using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcSessionsService.cs b/Common/src/gRPC/Services/GrpcSessionsService.cs index ddad9d9d7..04fd4e829 100644 --- a/Common/src/gRPC/Services/GrpcSessionsService.cs +++ b/Common/src/gRPC/Services/GrpcSessionsService.cs @@ -23,6 +23,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcSubmitterService.cs b/Common/src/gRPC/Services/GrpcSubmitterService.cs index 7c6c5f0d0..0ea46f946 100644 --- a/Common/src/gRPC/Services/GrpcSubmitterService.cs +++ b/Common/src/gRPC/Services/GrpcSubmitterService.cs @@ -22,6 +22,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcTasksService.cs b/Common/src/gRPC/Services/GrpcTasksService.cs index 0118b5979..7f5527edb 100644 --- a/Common/src/gRPC/Services/GrpcTasksService.cs +++ b/Common/src/gRPC/Services/GrpcTasksService.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/Submitter.cs b/Common/src/gRPC/Services/Submitter.cs index d9ac4f932..a41d2f575 100644 --- a/Common/src/gRPC/Services/Submitter.cs +++ b/Common/src/gRPC/Services/Submitter.cs @@ -27,7 +27,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Common/tests/AdapterLoading/AdapterLoadingTest.cs b/Common/tests/AdapterLoading/AdapterLoadingTest.cs index 87e60586b..24b10d408 100644 --- a/Common/tests/AdapterLoading/AdapterLoadingTest.cs +++ b/Common/tests/AdapterLoading/AdapterLoadingTest.cs @@ -67,8 +67,9 @@ public static void BuildServiceProvider(Dictionary config) configuration.AddInMemoryCollection(config); var serviceCollection = new ServiceCollection(); - serviceCollection.AddQueue(configuration, - logger); + serviceCollection.AddAdapter(configuration, + nameof(Components.QueueAdaptorSettings), + logger); serviceCollection.BuildServiceProvider(); } diff --git a/Common/tests/Helpers/SimpleObjectStorage.cs b/Common/tests/Helpers/SimpleObjectStorage.cs index 20226720c..677d10d1f 100644 --- a/Common/tests/Helpers/SimpleObjectStorage.cs +++ b/Common/tests/Helpers/SimpleObjectStorage.cs @@ -52,11 +52,4 @@ public IAsyncEnumerable GetValuesAsync(string key, public Task TryDeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) => Task.CompletedTask; - - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => new List - { - "key1", - "key2", - }.ToAsyncEnumerable(); } diff --git a/Common/tests/Helpers/TestDatabaseProvider.cs b/Common/tests/Helpers/TestDatabaseProvider.cs index 388fe18c0..b0705d960 100644 --- a/Common/tests/Helpers/TestDatabaseProvider.cs +++ b/Common/tests/Helpers/TestDatabaseProvider.cs @@ -107,10 +107,6 @@ public TestDatabaseProvider(Action? collectionConfigurato $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, diff --git a/Common/tests/Helpers/TestPollingAgentProvider.cs b/Common/tests/Helpers/TestPollingAgentProvider.cs index f5b110d7c..4c10d42d7 100644 --- a/Common/tests/Helpers/TestPollingAgentProvider.cs +++ b/Common/tests/Helpers/TestPollingAgentProvider.cs @@ -89,10 +89,6 @@ public TestPollingAgentProvider(IWorkerStreamHandler workerStreamHandler) $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, diff --git a/Common/tests/Helpers/TestPollsterProvider.cs b/Common/tests/Helpers/TestPollsterProvider.cs index f6c1e9533..4d250bc81 100644 --- a/Common/tests/Helpers/TestPollsterProvider.cs +++ b/Common/tests/Helpers/TestPollsterProvider.cs @@ -113,10 +113,6 @@ public TestPollsterProvider(IWorkerStreamHandler workerStreamHandler, $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, diff --git a/Common/tests/Helpers/TestTaskHandlerProvider.cs b/Common/tests/Helpers/TestTaskHandlerProvider.cs index ee8291db9..91753bdd8 100644 --- a/Common/tests/Helpers/TestTaskHandlerProvider.cs +++ b/Common/tests/Helpers/TestTaskHandlerProvider.cs @@ -109,10 +109,6 @@ public TestTaskHandlerProvider(IWorkerStreamHandler workerStreamHandler, $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, diff --git a/Common/tests/Pollster/PollsterTest.cs b/Common/tests/Pollster/PollsterTest.cs index f9ea55cea..f36b4d238 100644 --- a/Common/tests/Pollster/PollsterTest.cs +++ b/Common/tests/Pollster/PollsterTest.cs @@ -25,7 +25,7 @@ using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Pollster/TaskHandlerTest.cs b/Common/tests/Pollster/TaskHandlerTest.cs index 3867092d6..996d01d8c 100644 --- a/Common/tests/Pollster/TaskHandlerTest.cs +++ b/Common/tests/Pollster/TaskHandlerTest.cs @@ -29,6 +29,7 @@ using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Meter; @@ -1796,9 +1797,6 @@ public IAsyncEnumerable GetValuesAsync(string key, public Task TryDeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) => Task.CompletedTask; - - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); } [Test] diff --git a/Common/tests/Submitter/GrpcSubmitterServiceTests.cs b/Common/tests/Submitter/GrpcSubmitterServiceTests.cs index 312fe17ad..ccc8ff0b9 100644 --- a/Common/tests/Submitter/GrpcSubmitterServiceTests.cs +++ b/Common/tests/Submitter/GrpcSubmitterServiceTests.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs b/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs index c1f0f4bc6..0e04b6ad4 100644 --- a/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs +++ b/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Submitter/SubmitterTests.cs b/Common/tests/Submitter/SubmitterTests.cs index fe9f173de..25776c10b 100644 --- a/Common/tests/Submitter/SubmitterTests.cs +++ b/Common/tests/Submitter/SubmitterTests.cs @@ -94,10 +94,6 @@ public async Task SetUp() $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 72248c1b4..fd644238a 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -23,7 +23,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; diff --git a/Common/tests/TestBase/ResultTableTestBase.cs b/Common/tests/TestBase/ResultTableTestBase.cs index 797fd1976..01381592e 100644 --- a/Common/tests/TestBase/ResultTableTestBase.cs +++ b/Common/tests/TestBase/ResultTableTestBase.cs @@ -23,6 +23,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Common/tests/TestBase/TaskTableTestBase.cs b/Common/tests/TestBase/TaskTableTestBase.cs index c197133e5..c84fa8030 100644 --- a/Common/tests/TestBase/TaskTableTestBase.cs +++ b/Common/tests/TestBase/TaskTableTestBase.cs @@ -32,6 +32,7 @@ using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC; using ArmoniK.Core.Common.gRPC.Convertors; diff --git a/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj b/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj index de55cc27f..ccc01cead 100644 --- a/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj +++ b/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj @@ -36,10 +36,7 @@ - - - diff --git a/Compute/PollingAgent/src/Program.cs b/Compute/PollingAgent/src/Program.cs index 9d552849f..80444fdf3 100644 --- a/Compute/PollingAgent/src/Program.cs +++ b/Compute/PollingAgent/src/Program.cs @@ -22,14 +22,12 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Adapters.LocalStorage; using ArmoniK.Core.Adapters.MongoDB; -using ArmoniK.Core.Adapters.Redis; -using ArmoniK.Core.Adapters.S3; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Meter; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Pollster.TaskProcessingChecker; @@ -90,14 +88,12 @@ public static async Task Main(string[] args) .AddArmoniKWorkerConnection(builder.Configuration) .AddMongoComponents(builder.Configuration, logger.GetLogger()) - .AddQueue(builder.Configuration, - logger.GetLogger()) - .AddRedis(builder.Configuration, - logger.GetLogger()) - .AddS3(builder.Configuration, - logger.GetLogger()) - .AddLocalStorage(builder.Configuration, - logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.QueueAdaptorSettings), + logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()) .AddHostedService() .AddHostedService() .AddHostedService() diff --git a/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj b/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj index 10d4781b6..53290c993 100644 --- a/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj +++ b/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj @@ -35,10 +35,7 @@ - - - diff --git a/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj b/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj index 6610aa411..5a70d5b96 100644 --- a/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj +++ b/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj @@ -37,10 +37,7 @@ - - - diff --git a/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj b/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj index 9177894ff..72201f23b 100644 --- a/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj +++ b/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj @@ -38,9 +38,6 @@ - - - diff --git a/Control/Submitter/src/Program.cs b/Control/Submitter/src/Program.cs index 892906ca5..4eb53d567 100644 --- a/Control/Submitter/src/Program.cs +++ b/Control/Submitter/src/Program.cs @@ -22,16 +22,14 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Adapters.LocalStorage; using ArmoniK.Core.Adapters.MongoDB; -using ArmoniK.Core.Adapters.Redis; -using ArmoniK.Core.Adapters.S3; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.gRPC; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Meter; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; @@ -88,14 +86,12 @@ public static async Task Main(string[] args) .AddHttpClient() .AddMongoComponents(builder.Configuration, logger.GetLogger()) - .AddQueue(builder.Configuration, - logger.GetLogger()) - .AddRedis(builder.Configuration, - logger.GetLogger()) - .AddLocalStorage(builder.Configuration, - logger.GetLogger()) - .AddS3(builder.Configuration, - logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.QueueAdaptorSettings), + logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()) .AddSingleton() .AddSingletonWithHealthCheck(nameof(ExceptionInterceptor)) .AddOption(builder.Configuration, diff --git a/Dockerfile b/Dockerfile index f8918d636..02a57a11b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,9 @@ RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Amqp/src/ArmoniK.Core.Adapters.A RUN dotnet restore -a "${TARGETARCH}" "Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj" # git ls-tree -r HEAD --name-only --full-tree | grep "csproj$" | xargs -I % sh -c "export D=\$(dirname %) ; echo COPY [\\\"\$D\\\", \\\"\$D\\\"]" COPY ["Adaptors/Amqp/src", "Adaptors/Amqp/src"] @@ -72,6 +75,15 @@ RUN dotnet publish "ArmoniK.Core.Adapters.Amqp.csproj" -a "${TARGETARCH}" --no-r WORKDIR /src/Adaptors/RabbitMQ/src RUN dotnet publish "ArmoniK.Core.Adapters.RabbitMQ.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/rabbit /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION +WORKDIR /src/Adaptors/LocalStorage/src +RUN dotnet publish "ArmoniK.Core.Adapters.LocalStorage.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/local_storage /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/Redis/src +RUN dotnet publish "ArmoniK.Core.Adapters.Redis.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/redis /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/S3/src +RUN dotnet publish "ArmoniK.Core.Adapters.S3.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/s3 /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + WORKDIR /src/Compute/PollingAgent/src RUN dotnet publish "ArmoniK.Core.Compute.PollingAgent.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/polling_agent /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION @@ -94,6 +106,12 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /adapters/object/local_storage +COPY --from=build /app/publish/local_storage . +WORKDIR /adapters/object/redis +COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/s3 +COPY --from=build /app/publish/s3 . WORKDIR /app COPY --from=build /app/publish/polling_agent . ENV ASPNETCORE_URLS http://+:1080 @@ -126,6 +144,12 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /adapters/object/local_storage +COPY --from=build /app/publish/local_storage . +WORKDIR /adapters/object/redis +COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/s3 +COPY --from=build /app/publish/s3 . WORKDIR /app COPY --from=build /app/publish/submitter . ENV ASPNETCORE_URLS http://+:1080, http://+:1081 diff --git a/terraform/modules/storage/object/local/output.tf b/terraform/modules/storage/object/local/output.tf index 0892e2b89..bb9ad423b 100644 --- a/terraform/modules/storage/object/local/output.tf +++ b/terraform/modules/storage/object/local/output.tf @@ -1,7 +1,8 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.LocalStorage.ObjectStorage", - "LocalStorage__Path" = var.local_path + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.LocalStorage.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/local_storage/ArmoniK.Core.Adapters.LocalStorage.dll" + "LocalStorage__Path" = var.local_path }) } diff --git a/terraform/modules/storage/object/minio/outputs.tf b/terraform/modules/storage/object/minio/outputs.tf index 6f9947d26..ad1c280ad 100644 --- a/terraform/modules/storage/object/minio/outputs.tf +++ b/terraform/modules/storage/object/minio/outputs.tf @@ -1,11 +1,12 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.S3.ObjectStorage", - "S3__EndpointUrl" = "http://${var.host}:${var.port}" - "S3__BucketName" = var.bucket_name - "S3__Login" = var.login - "S3__Password" = var.password - "S3__MustForcePathStyle" = true + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.S3.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/s3/ArmoniK.Core.Adapters.S3.dll" + "S3__EndpointUrl" = "http://${var.host}:${var.port}" + "S3__BucketName" = var.bucket_name + "S3__Login" = var.login + "S3__Password" = var.password + "S3__MustForcePathStyle" = true }) } diff --git a/terraform/modules/storage/object/redis/outputs.tf b/terraform/modules/storage/object/redis/outputs.tf index 7ffd47b7f..82b09244f 100644 --- a/terraform/modules/storage/object/redis/outputs.tf +++ b/terraform/modules/storage/object/redis/outputs.tf @@ -1,7 +1,8 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.Redis.ObjectStorage", - "Redis__EndpointUrl" = "object:${var.exposed_port}" + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Redis.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/redis/ArmoniK.Core.Adapters.Redis.dll" + "Redis__EndpointUrl" = "object:${var.exposed_port}" }) } output "volumes" { From e8fa5f6a7996e762fb1725dbd7ce0b49ee96b532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Wed, 6 Nov 2024 11:34:09 +0100 Subject: [PATCH 02/15] ci: build all solution instead of test project only --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba30f4d73..5ccd4a441 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,12 +72,12 @@ jobs: - name: Dotnet Restore run: | - MONITOR_PREFIX="monitor/restore/" MONITOR_CD=${{ matrix.projects }} tools/retry.sh -w 60 -- tools/monitor.sh \ + MONITOR_PREFIX="monitor/restore/" tools/retry.sh -w 60 -- tools/monitor.sh \ dotnet restore - name: Dotnet Build run: | - MONITOR_PREFIX="monitor/build/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \ + MONITOR_PREFIX="monitor/build/" tools/monitor.sh \ dotnet build - name: Run tests From b08f5008ddc0e0c12c1fc335dd31ce78bf526092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Wed, 6 Nov 2024 14:00:19 +0100 Subject: [PATCH 03/15] test: add missing IObjectStorage --- Adaptors/Memory/src/ObjectStorage.cs | 97 +++++++++++++++++++ Adaptors/Memory/tests/ObjectStorageTests.cs | 39 ++++++++ Common/tests/Helpers/TestDatabaseProvider.cs | 2 + .../tests/Helpers/TestPollingAgentProvider.cs | 1 + Common/tests/Helpers/TestPollsterProvider.cs | 1 + .../tests/Helpers/TestTaskHandlerProvider.cs | 2 + Common/tests/Submitter/SubmitterTests.cs | 7 +- 7 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 Adaptors/Memory/src/ObjectStorage.cs create mode 100644 Adaptors/Memory/tests/ObjectStorageTests.cs diff --git a/Adaptors/Memory/src/ObjectStorage.cs b/Adaptors/Memory/src/ObjectStorage.cs new file mode 100644 index 000000000..e8fc26685 --- /dev/null +++ b/Adaptors/Memory/src/ObjectStorage.cs @@ -0,0 +1,97 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; +using ArmoniK.Core.Common.Storage; +using ArmoniK.Utils; + +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ArmoniK.Core.Adapters.Memory; + +public class ObjectStorage : IObjectStorage +{ + private readonly ConcurrentDictionary store_ = new(); + private bool isInitialized_; + + /// + public Task Init(CancellationToken cancellationToken) + { + isInitialized_ = true; + return Task.CompletedTask; + } + + /// + public Task Check(HealthCheckTag tag) + => Task.FromResult(isInitialized_ + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy()); + + public async Task AddOrUpdateAsync(string key, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + { + var array = new List(); + + await foreach (var val in valueChunks.WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + array.AddRange(val.ToArray()); + } + + store_[key] = array.ToArray(); + + return array.Count; + } + +#pragma warning disable CS1998 + public async IAsyncEnumerable GetValuesAsync(string key, +#pragma warning restore CS1998 + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (!store_.TryGetValue(key, + out var value)) + { + throw new ObjectDataNotFoundException(); + } + + foreach (var chunk in value.ToChunks(100)) + { + yield return chunk; + } + } + + public Task TryDeleteAsync(IEnumerable keys, + CancellationToken cancellationToken = default) + { + foreach (var key in keys) + { + store_.TryRemove(key, + out _); + } + + return Task.CompletedTask; + } +} diff --git a/Adaptors/Memory/tests/ObjectStorageTests.cs b/Adaptors/Memory/tests/ObjectStorageTests.cs new file mode 100644 index 000000000..2a813a797 --- /dev/null +++ b/Adaptors/Memory/tests/ObjectStorageTests.cs @@ -0,0 +1,39 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using ArmoniK.Core.Common.Tests.TestBase; + +using NUnit.Framework; + +namespace ArmoniK.Core.Adapters.Memory.Tests; + +[TestFixture] +public class ObjectStorageTests : ObjectStorageTestBase +{ + public override void TearDown() + { + ObjectStorage = null; + RunTests = false; + } + + + protected override void GetObjectStorageInstance() + { + ObjectStorage = new ObjectStorage(); + RunTests = true; + } +} diff --git a/Common/tests/Helpers/TestDatabaseProvider.cs b/Common/tests/Helpers/TestDatabaseProvider.cs index b0705d960..0ce3c41c0 100644 --- a/Common/tests/Helpers/TestDatabaseProvider.cs +++ b/Common/tests/Helpers/TestDatabaseProvider.cs @@ -21,6 +21,7 @@ using System.Threading; using ArmoniK.Api.Common.Options; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Adapters.MongoDB.Common; using ArmoniK.Core.Common.Auth.Authentication; @@ -136,6 +137,7 @@ public TestDatabaseProvider(Action? collectionConfigurato out _) .Configure(o => o.CopyFrom(AuthenticatorOptions.DefaultNoAuth)) .AddLogging() + .AddSingleton() .AddSingleton(loggerProvider.CreateLogger("root")) .AddSingleton(ActivitySource) .AddSingleton(_ => client_); diff --git a/Common/tests/Helpers/TestPollingAgentProvider.cs b/Common/tests/Helpers/TestPollingAgentProvider.cs index 4c10d42d7..ef15e8dcc 100644 --- a/Common/tests/Helpers/TestPollingAgentProvider.cs +++ b/Common/tests/Helpers/TestPollingAgentProvider.cs @@ -113,6 +113,7 @@ public TestPollingAgentProvider(IWorkerStreamHandler workerStreamHandler) .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton(workerStreamHandler); var computePlanOptions = builder.Configuration.GetRequiredValue(ComputePlane.SettingSection); diff --git a/Common/tests/Helpers/TestPollsterProvider.cs b/Common/tests/Helpers/TestPollsterProvider.cs index 4d250bc81..859dd2966 100644 --- a/Common/tests/Helpers/TestPollsterProvider.cs +++ b/Common/tests/Helpers/TestPollsterProvider.cs @@ -176,6 +176,7 @@ acquireTimeout is null .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddOption(builder.Configuration, Injection.Options.Pollster.SettingSection) diff --git a/Common/tests/Helpers/TestTaskHandlerProvider.cs b/Common/tests/Helpers/TestTaskHandlerProvider.cs index 91753bdd8..b03035556 100644 --- a/Common/tests/Helpers/TestTaskHandlerProvider.cs +++ b/Common/tests/Helpers/TestTaskHandlerProvider.cs @@ -22,6 +22,7 @@ using System.Threading; using ArmoniK.Api.Common.Options; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Base; using ArmoniK.Core.Common.gRPC.Services; @@ -156,6 +157,7 @@ public TestTaskHandlerProvider(IWorkerStreamHandler workerStreamHandler, .AddOption(builder.Configuration, Injection.Options.Pollster.SettingSection) .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/Common/tests/Submitter/SubmitterTests.cs b/Common/tests/Submitter/SubmitterTests.cs index 25776c10b..5bba0080d 100644 --- a/Common/tests/Submitter/SubmitterTests.cs +++ b/Common/tests/Submitter/SubmitterTests.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Options; using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Base; using ArmoniK.Core.Common.Exceptions; @@ -122,6 +123,7 @@ public async Task SetUp() .AddSingleton(client_) .AddLogging(builder => builder.AddProvider(loggerProvider)) .AddSingleton() + .AddSingleton() .AddOption(configuration, Injection.Options.Submitter.SettingSection) .AddSingleton(pushQueueStorage_); @@ -965,10 +967,7 @@ await submitter_.TryGetResult(resultRequest, Assert.AreEqual(ResultReply.TypeOneofCase.Result, writer.Messages[1] .TypeCase); - Assert.AreEqual(ResultReply.TypeOneofCase.Result, - writer.Messages[2] - .TypeCase); - Assert.IsTrue(writer.Messages[2] + Assert.IsTrue(writer.Messages[1] .Result.DataComplete); } From e3e044ae7b28459649af4db63aac1280007af0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Wed, 6 Nov 2024 18:51:02 +0100 Subject: [PATCH 04/15] feat: Allow ObjectStorages to create an Opaque Id for the Results --- Adaptors/LocalStorage/src/ObjectBuilder.cs | 1 - Adaptors/LocalStorage/src/ObjectStorage.cs | 21 +-- Adaptors/Memory/src/ObjectStorage.cs | 22 ++-- .../Table/DataModel/ResultDataModelMapping.cs | 4 +- Adaptors/MongoDB/tests/BsonSerializerTest.cs | 2 +- Adaptors/Redis/src/ObjectBuilder.cs | 1 - Adaptors/Redis/src/ObjectStorage.cs | 23 ++-- Adaptors/Redis/tests/ObjectStorageTests.cs | 2 +- Adaptors/S3/src/ObjectBuilder.cs | 1 - Adaptors/S3/src/ObjectStorage.cs | 29 ++-- Adaptors/S3/tests/ObjectStorageTests.cs | 2 +- Base/src/IObjectStorage.cs | 22 ++-- Common/src/Pollster/Agent.cs | 98 +++++++------- Common/src/Pollster/DataPrefetcher.cs | 35 ++--- Common/src/Storage/Result.cs | 4 +- Common/src/Storage/ResultLifeCycleHelper.cs | 9 +- Common/src/Storage/ResultTableExtensions.cs | 52 ++------ .../src/gRPC/Services/GrpcResultsService.cs | 88 ++++++++----- Common/src/gRPC/Services/Submitter.cs | 79 ++++++----- Common/tests/Helpers/SimpleObjectStorage.cs | 21 ++- Common/tests/Helpers/TestDatabaseProvider.cs | 1 + Common/tests/Pollster/AgentTest.cs | 64 +++++---- Common/tests/Pollster/DataPrefetcherTest.cs | 124 ++++++++++++++---- Common/tests/Pollster/TaskHandlerTest.cs | 13 +- Common/tests/TaskLifeCycleHelperTest.cs | 12 +- .../tests/TestBase/ObjectStorageTestBase.cs | 67 +++++----- Common/tests/TestBase/ResultTableTestBase.cs | 51 +++---- 27 files changed, 475 insertions(+), 373 deletions(-) diff --git a/Adaptors/LocalStorage/src/ObjectBuilder.cs b/Adaptors/LocalStorage/src/ObjectBuilder.cs index 1a71d92e2..b29e4563b 100644 --- a/Adaptors/LocalStorage/src/ObjectBuilder.cs +++ b/Adaptors/LocalStorage/src/ObjectBuilder.cs @@ -16,7 +16,6 @@ // along with this program. If not, see . using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Utils; using JetBrains.Annotations; diff --git a/Adaptors/LocalStorage/src/ObjectStorage.cs b/Adaptors/LocalStorage/src/ObjectStorage.cs index 8673201eb..ea8e44174 100644 --- a/Adaptors/LocalStorage/src/ObjectStorage.cs +++ b/Adaptors/LocalStorage/src/ObjectStorage.cs @@ -22,9 +22,9 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; -using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -93,13 +93,13 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { + var id = Guid.NewGuid(); long size = 0; var filename = Path.Combine(path_, - key); + id.ToString()); // Write to temporary file @@ -133,13 +133,15 @@ public async Task AddOrUpdateAsync(string await file.FlushAsync(cancellationToken) .ConfigureAwait(false); - return size; + return (id.ToByteArray(), size); } /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var key = new Guid(id).ToString(); + var filename = Path.Combine(path_, key); @@ -183,11 +185,12 @@ public async IAsyncEnumerable GetValuesAsync(string } /// - public async Task TryDeleteAsync(IEnumerable keys, + public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) { - foreach (var key in keys) + foreach (var id in ids) { + var key = new Guid(id).ToString(); await TryDeleteAsync(key, cancellationToken) .ConfigureAwait(false); diff --git a/Adaptors/Memory/src/ObjectStorage.cs b/Adaptors/Memory/src/ObjectStorage.cs index e8fc26685..b9dd2df41 100644 --- a/Adaptors/Memory/src/ObjectStorage.cs +++ b/Adaptors/Memory/src/ObjectStorage.cs @@ -22,9 +22,9 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; -using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -49,28 +49,30 @@ public Task Check(HealthCheckTag tag) ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy()); - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { var array = new List(); + var id = Guid.NewGuid(); + await foreach (var val in valueChunks.WithCancellation(cancellationToken) .ConfigureAwait(false)) { array.AddRange(val.ToArray()); } - store_[key] = array.ToArray(); + store_[id.ToString()] = array.ToArray(); - return array.Count; + return (id.ToByteArray(), array.Count); } #pragma warning disable CS1998 - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, #pragma warning restore CS1998 [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var key = new Guid(id).ToString(); if (!store_.TryGetValue(key, out var value)) { @@ -83,11 +85,13 @@ public async IAsyncEnumerable GetValuesAsync(string key, } } - public Task TryDeleteAsync(IEnumerable keys, + public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) { - foreach (var key in keys) + foreach (var id in ids) { + var key = new Guid(id).ToString(); + store_.TryRemove(key, out _); } diff --git a/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs b/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs index 1621b252c..f4ba0cd91 100644 --- a/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs +++ b/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs @@ -53,7 +53,7 @@ public ResultDataModelMapping() cm.MapProperty(nameof(Result.Size)) .SetIgnoreIfDefault(true) .SetDefaultValue(0); - cm.MapProperty(nameof(Result.Data)) + cm.MapProperty(nameof(Result.OpaqueId)) .SetIsRequired(true); cm.SetIgnoreExtraElements(true); cm.MapCreator(model => new Result(model.SessionId, @@ -65,7 +65,7 @@ public ResultDataModelMapping() model.DependentTasks, model.CreationDate, model.Size, - model.Data)); + model.OpaqueId)); }); } } diff --git a/Adaptors/MongoDB/tests/BsonSerializerTest.cs b/Adaptors/MongoDB/tests/BsonSerializerTest.cs index 0f5d39d83..213ed7aaf 100644 --- a/Adaptors/MongoDB/tests/BsonSerializerTest.cs +++ b/Adaptors/MongoDB/tests/BsonSerializerTest.cs @@ -119,7 +119,7 @@ public void SerializeResultDataModel() deserialized.DependentTasks); Assert.AreEqual(rdm.CreationDate, deserialized.CreationDate); - Assert.IsTrue(rdm.Data.SequenceEqual(deserialized.Data)); + Assert.IsTrue(rdm.OpaqueId.SequenceEqual(deserialized.OpaqueId)); } [Test] diff --git a/Adaptors/Redis/src/ObjectBuilder.cs b/Adaptors/Redis/src/ObjectBuilder.cs index ef702d91d..af19fec07 100644 --- a/Adaptors/Redis/src/ObjectBuilder.cs +++ b/Adaptors/Redis/src/ObjectBuilder.cs @@ -20,7 +20,6 @@ using System.Security.Cryptography.X509Certificates; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Utils; using JetBrains.Annotations; diff --git a/Adaptors/Redis/src/ObjectStorage.cs b/Adaptors/Redis/src/ObjectStorage.cs index dec9a91e8..010db5852 100644 --- a/Adaptors/Redis/src/ObjectStorage.cs +++ b/Adaptors/Redis/src/ObjectStorage.cs @@ -22,9 +22,9 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; -using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -86,12 +86,12 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { + var id = Guid.NewGuid(); long size = 0; - var storageNameKey = objectStorageName_ + key; + var storageNameKey = objectStorageName_ + id; var idx = 0; var taskList = new List(); @@ -110,13 +110,14 @@ public async Task AddOrUpdateAsync(string await taskList.WhenAll() .ConfigureAwait(false); - return size; + return (id.ToByteArray(), size); } /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var key = new Guid(id); var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); @@ -142,11 +143,11 @@ public async IAsyncEnumerable GetValuesAsync(string } /// - public async Task TryDeleteAsync(IEnumerable keys, + public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) - => await keys.ParallelForEach(key => TryDeleteAsync(key, - cancellationToken)) - .ConfigureAwait(false); + => await ids.ParallelForEach(key => TryDeleteAsync(new Guid(key).ToString(), + cancellationToken)) + .ConfigureAwait(false); private async Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) diff --git a/Adaptors/Redis/tests/ObjectStorageTests.cs b/Adaptors/Redis/tests/ObjectStorageTests.cs index db2544f84..cd43ef46f 100644 --- a/Adaptors/Redis/tests/ObjectStorageTests.cs +++ b/Adaptors/Redis/tests/ObjectStorageTests.cs @@ -19,7 +19,7 @@ using System.Collections.Generic; using System.IO; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Utils; diff --git a/Adaptors/S3/src/ObjectBuilder.cs b/Adaptors/S3/src/ObjectBuilder.cs index 9432a1709..17df4d945 100644 --- a/Adaptors/S3/src/ObjectBuilder.cs +++ b/Adaptors/S3/src/ObjectBuilder.cs @@ -18,7 +18,6 @@ using Amazon.S3; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Utils; using JetBrains.Annotations; diff --git a/Adaptors/S3/src/ObjectStorage.cs b/Adaptors/S3/src/ObjectStorage.cs index eac8a7015..572502aaf 100644 --- a/Adaptors/S3/src/ObjectStorage.cs +++ b/Adaptors/S3/src/ObjectStorage.cs @@ -27,9 +27,9 @@ using Amazon.S3.Model; using Amazon.S3.Util; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; -using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -95,9 +95,10 @@ public Task Check(HealthCheckTag tag) }; /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var key = new Guid(id); var objectStorageFullName = $"{objectStorageName_}{key}"; try @@ -163,23 +164,16 @@ await s3Client_.GetObjectAsync(options_.BucketName, } /// - public async Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - => await keys.ParallelForEach(key => TryDeleteAsync(key, - cancellationToken)) - .ConfigureAwait(false); - - /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { long[] sizeBox = { 0, }; - var objectStorageFullName = $"{objectStorageName_}{key}"; + var id = Guid.NewGuid(); + var objectStorageFullName = $"{objectStorageName_}{id.ToString()}"; logger_.LogDebug("Upload object"); var initRequest = new InitiateMultipartUploadRequest @@ -231,9 +225,16 @@ await s3Client_.AbortMultipartUploadAsync(abortMpuRequest, throw; } - return sizeBox[0]; + return (id.ToByteArray(), sizeBox[0]); } + /// + public async Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => await ids.ParallelForEach(key => TryDeleteAsync(new Guid(key).ToString(), + cancellationToken)) + .ConfigureAwait(false); + private async Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) { diff --git a/Adaptors/S3/tests/ObjectStorageTests.cs b/Adaptors/S3/tests/ObjectStorageTests.cs index 663282232..db6615666 100644 --- a/Adaptors/S3/tests/ObjectStorageTests.cs +++ b/Adaptors/S3/tests/ObjectStorageTests.cs @@ -15,9 +15,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Injection; using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Common.Utils; diff --git a/Base/src/IObjectStorage.cs b/Base/src/IObjectStorage.cs index 2be7f95c6..b1bd25a51 100644 --- a/Base/src/IObjectStorage.cs +++ b/Base/src/IObjectStorage.cs @@ -20,10 +20,9 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Base; using ArmoniK.Core.Base.Exceptions; -namespace ArmoniK.Core.Common.Storage; +namespace ArmoniK.Core.Base; /// /// Object Storage interface @@ -35,38 +34,39 @@ public interface IObjectStorage : IInitializable /// Add the given data in the storage at the given key /// Update data if it already exists /// - /// Key representing the object /// Chunks of data that will be stored in the Object Storage /// Token used to cancel the execution of the method /// - /// The size of the object that has been uploaded. + /// The opaque unique identifier representing the object and the size of the object that has been uploaded. /// + /// + /// Opaque ID will be used to refer to the object. The implementation of this interface should generate it. + /// /// the key is not found - Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default); + Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default); /// /// Get object in the Object Storage /// - /// Key representing the object to be retrieved + /// Opaque unique identifier representing the object /// Token used to cancel the execution of the method /// /// Byte arrays representing the object chunked /// /// the key is not found - IAsyncEnumerable GetValuesAsync(string key, + IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default); /// /// Delete data in the object storage /// - /// Keys representing the objects to delete + /// Opaque unique identifiers representing the objects to delete /// Token used to cancel the execution of the method /// /// Task representing the asynchronous execution of the method /// /// the key is not found - Task TryDeleteAsync(IEnumerable keys, + Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default); } diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index fd95c9724..b0cab6aae 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -27,6 +27,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Core.Base; using ArmoniK.Core.Base.Exceptions; +using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -45,16 +46,16 @@ namespace ArmoniK.Core.Common.Pollster; /// public sealed class Agent : IAgent { - private readonly List createdTasks_; - private readonly ILogger logger_; - private readonly IObjectStorage objectStorage_; - private readonly IPushQueueStorage pushQueueStorage_; - private readonly IResultTable resultTable_; - private readonly Dictionary sentResults_; - private readonly SessionData sessionData_; - private readonly ISubmitter submitter_; - private readonly TaskData taskData_; - private readonly ITaskTable taskTable_; + private readonly List createdTasks_; + private readonly ILogger logger_; + private readonly IObjectStorage objectStorage_; + private readonly IPushQueueStorage pushQueueStorage_; + private readonly IResultTable resultTable_; + private readonly Dictionary sentResults_; + private readonly SessionData sessionData_; + private readonly ISubmitter submitter_; + private readonly TaskData taskData_; + private readonly ITaskTable taskTable_; /// /// Initializes a new instance of the @@ -64,8 +65,8 @@ public sealed class Agent : IAgent /// Interface to put tasks in the queue /// Interface to manage result states /// Interface to manage task states - /// Data of the session - /// Data of the task + /// OpaqueId of the session + /// OpaqueId of the task /// Shared folder between Agent and Worker /// Token send to the worker to identify the running task /// Logger used to produce logs for this class @@ -87,7 +88,7 @@ public Agent(ISubmitter submitter, taskTable_ = taskTable; logger_ = logger; createdTasks_ = new List(); - sentResults_ = new Dictionary(); + sentResults_ = new Dictionary(); sessionData_ = sessionData; taskData_ = taskData; Folder = folder; @@ -120,11 +121,12 @@ await submitter_.FinalizeTaskCreation(createdTasks_, cancellationToken) .ConfigureAwait(false); - foreach (var (result, size) in sentResults_) + foreach (var (result, (id, size)) in sentResults_) { await resultTable_.CompleteResult(taskData_.SessionId, result, size, + id, cancellationToken) .ConfigureAwait(false); } @@ -155,29 +157,32 @@ public async Task GetResourceData(string token, ThrowIfInvalidToken(token); + try { - await using (var fs = new FileStream(Path.Combine(Folder, - resultId), - FileMode.OpenOrCreate)) + var opaqueId = (await resultTable_.GetResult(resultId, + cancellationToken) + .ConfigureAwait(false)).OpaqueId; + + await using var fs = new FileStream(Path.Combine(Folder, + resultId), + FileMode.OpenOrCreate); + await using var w = new BinaryWriter(fs); + await foreach (var chunk in objectStorage_.GetValuesAsync(opaqueId, + cancellationToken) + .ConfigureAwait(false)) { - await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(resultId, - cancellationToken) - .ConfigureAwait(false)) - { - w.Write(chunk); - } + w.Write(chunk); } return resultId; } - catch (ObjectDataNotFoundException) + catch (Exception ex) when (ex is ResultNotFoundException or ObjectDataNotFoundException) { throw new RpcException(new Status(StatusCode.NotFound, - "Data not found"), - "Data not found"); + "ResultId not found"), + "ResultId not found"); } } @@ -260,18 +265,22 @@ public async Task> CreateResults(string { ThrowIfInvalidToken(token); + var local = new Dictionary(); + var results = await requests.Select(async rc => { var resultId = Guid.NewGuid() .ToString(); - var size = await objectStorage_.AddOrUpdateAsync(resultId, - new List> - { - rc.data, - }.ToAsyncEnumerable(), - cancellationToken) - .ConfigureAwait(false); + var add = await objectStorage_.AddOrUpdateAsync(new List> + { + rc.data, + }.ToAsyncEnumerable(), + cancellationToken) + .ConfigureAwait(false); + + local.Add(resultId, + add); return new Result(rc.request.SessionId, resultId, @@ -281,7 +290,7 @@ public async Task> CreateResults(string ResultStatus.Created, new List(), DateTime.UtcNow, - size, + 0, Array.Empty()); }) .WhenAll() @@ -291,10 +300,10 @@ await resultTable_.Create(results, cancellationToken) .ConfigureAwait(false); - foreach (var result in results) + foreach (var result in local) { - sentResults_.Add(result.ResultId, - result.Size); + sentResults_.Add(result.Key, + result.Value); } return results; @@ -315,19 +324,16 @@ public async Task> NotifyResultData(string toke using var r = new BinaryReader(fs); var channel = Channel.CreateUnbounded>(); - var add = objectStorage_.AddOrUpdateAsync(result, - channel.Reader.ReadAllAsync(cancellationToken), - cancellationToken); + var addTask = objectStorage_.AddOrUpdateAsync(channel.Reader.ReadAllAsync(cancellationToken), + cancellationToken); - long size = 0; - int read; + int read; do { var buffer = new byte[PayloadConfiguration.MaxChunkSize]; read = r.Read(buffer, 0, PayloadConfiguration.MaxChunkSize); - size += read; if (read > 0) { await channel.Writer.WriteAsync(buffer.AsMemory(0, @@ -339,9 +345,9 @@ await channel.Writer.WriteAsync(buffer.AsMemory(0, channel.Writer.Complete(); - await add.ConfigureAwait(false); + var add = await addTask.ConfigureAwait(false); sentResults_.Add(result, - size); + add); } return resultIds; diff --git a/Common/src/Pollster/DataPrefetcher.cs b/Common/src/Pollster/DataPrefetcher.cs index 26a17600c..a6c658589 100644 --- a/Common/src/Pollster/DataPrefetcher.cs +++ b/Common/src/Pollster/DataPrefetcher.cs @@ -15,8 +15,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -39,6 +41,7 @@ public class DataPrefetcher : IInitializable private readonly ActivitySource? activitySource_; private readonly ILogger logger_; private readonly IObjectStorage objectStorage_; + private readonly IResultTable resultTable_; private bool isInitialized_; @@ -46,13 +49,16 @@ public class DataPrefetcher : IInitializable /// Create data prefetcher for tasks /// /// Interface to manage data + /// Interface to manage results metadata /// Activity source for tracing /// Logger used to print logs public DataPrefetcher(IObjectStorage objectStorage, + IResultTable resultTable, ActivitySource? activitySource, ILogger logger) { objectStorage_ = objectStorage; + resultTable_ = resultTable; logger_ = logger; activitySource_ = activitySource; } @@ -93,28 +99,27 @@ public async Task PrefetchDataAsync(TaskData taskData, activity?.AddEvent(new ActivityEvent("Load payload")); + var dependencies = new List + { + taskData.PayloadId, + }; + dependencies.AddRange(taskData.DataDependencies); - await using (var fs = new FileStream(Path.Combine(folder, - taskData.PayloadId), - FileMode.OpenOrCreate)) - { - await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(taskData.PayloadId, - cancellationToken) - .ConfigureAwait(false)) - { - w.Write(chunk); - } - } + var ids = await resultTable_.GetResults(taskData.SessionId, + dependencies, + cancellationToken) + .Select(r => (r.ResultId, r.OpaqueId)) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); - foreach (var dataDependency in taskData.DataDependencies) + foreach (var (resultId, opaqueId) in ids) { await using var fs = new FileStream(Path.Combine(folder, - dataDependency), + resultId), FileMode.OpenOrCreate); await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(dataDependency, + await foreach (var chunk in objectStorage_.GetValuesAsync(opaqueId, cancellationToken) .ConfigureAwait(false)) { diff --git a/Common/src/Storage/Result.cs b/Common/src/Storage/Result.cs index f0975b352..bb62abd48 100644 --- a/Common/src/Storage/Result.cs +++ b/Common/src/Storage/Result.cs @@ -31,7 +31,7 @@ namespace ArmoniK.Core.Common.Storage; /// List of tasks that depend on this result. /// Date of creation of the current object. /// Size of the result. -/// Data for the current +/// Opaque Identifier used by the object storage to refer to this result's data public record Result(string SessionId, string ResultId, string Name, @@ -41,7 +41,7 @@ public record Result(string SessionId, List DependentTasks, DateTime CreationDate, long Size, - byte[] Data) + byte[] OpaqueId) { /// /// Creates a copy of a and modify it according to given updates diff --git a/Common/src/Storage/ResultLifeCycleHelper.cs b/Common/src/Storage/ResultLifeCycleHelper.cs index 81e630e40..36c8d22d9 100644 --- a/Common/src/Storage/ResultLifeCycleHelper.cs +++ b/Common/src/Storage/ResultLifeCycleHelper.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Utils; using Microsoft.Extensions.Logging; @@ -156,8 +157,8 @@ public static async Task PurgeResultsAsync(IResultTable resultTable, string sessionId, CancellationToken cancellationToken) { - await foreach (var ids in resultTable.GetResults(result => result.SessionId == sessionId, - result => result.ResultId, + await foreach (var ids in resultTable.GetResults(result => result.SessionId == sessionId && result.OpaqueId.Length > 0, + result => result.OpaqueId, cancellationToken) .ToChunksAsync(500, Timeout.InfiniteTimeSpan, @@ -171,7 +172,9 @@ await objectStorage.TryDeleteAsync(ids, await resultTable.UpdateManyResults(result => result.SessionId == sessionId, new UpdateDefinition().Set(result => result.Status, - ResultStatus.DeletedData), + ResultStatus.DeletedData) + .Set(result => result.OpaqueId, + Array.Empty()), cancellationToken) .ConfigureAwait(false); } diff --git a/Common/src/Storage/ResultTableExtensions.cs b/Common/src/Storage/ResultTableExtensions.cs index 5475a6432..cbb0506c8 100644 --- a/Common/src/Storage/ResultTableExtensions.cs +++ b/Common/src/Storage/ResultTableExtensions.cs @@ -101,6 +101,7 @@ public static Task BulkUpdateResults(this IResultTable /// id of the session containing the results /// Id of the result to complete /// Size of the result to complete + /// Opaque unique identifier representing the object /// Token used to cancel the execution of the method /// /// The new version of the result metadata @@ -110,13 +111,16 @@ public static async Task CompleteResult(this IResultTable resultTable, string sessionId, string resultId, long size, + byte[] opaqueId, CancellationToken cancellationToken = default) { var result = await resultTable.UpdateOneResult(resultId, new UpdateDefinition().Set(data => data.Status, ResultStatus.Completed) .Set(data => data.Size, - size), + size) + .Set(data => data.OpaqueId, + opaqueId), cancellationToken) .ConfigureAwait(false); @@ -128,49 +132,10 @@ public static async Task CompleteResult(this IResultTable resultTable, { Status = ResultStatus.Completed, Size = size, + OpaqueId = opaqueId, }; } - /// - /// Update result with small payload - /// - /// Interface to manage results - /// id of the session containing the results - /// id of the task owning the result - /// id of the result to be modified - /// payload data - /// Token used to cancel the execution of the method - /// - /// Task representing the asynchronous execution of the method - /// - public static async Task SetResult(this IResultTable resultTable, - string sessionId, - string ownerTaskId, - string resultId, - byte[] smallPayload, - CancellationToken cancellationToken = default) - { - var count = await resultTable.UpdateManyResults(result => result.ResultId == resultId && result.OwnerTaskId == ownerTaskId, - new UpdateDefinition().Set(result => result.Status, - ResultStatus.Completed) - .Set(data => data.Size, - smallPayload.Length) - .Set(result => result.Data, - smallPayload), - cancellationToken) - .ConfigureAwait(false); - - resultTable.Logger.LogDebug("Update {result} from {owner} to {status}", - resultId, - ownerTaskId, - ResultStatus.Completed); - - if (count == 0) - { - throw new ResultNotFoundException($"Result '{resultId}' was not found for '{ownerTaskId}'"); - } - } - /// /// Update result /// @@ -188,13 +153,16 @@ public static async Task SetResult(this IResultTable resultTable, string ownerTaskId, string resultId, long size, + byte[] opaqueId, CancellationToken cancellationToken = default) { var count = await resultTable.UpdateManyResults(result => result.ResultId == resultId && result.OwnerTaskId == ownerTaskId, new UpdateDefinition().Set(result => result.Status, ResultStatus.Completed) .Set(result => result.Size, - size), + size) + .Set(result => result.OpaqueId, + opaqueId), cancellationToken) .ConfigureAwait(false); diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index 5c2bc96ef..1c4e7ac56 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -172,35 +172,36 @@ public override async Task CreateResults(CreateResultsReq var resultId = Guid.NewGuid() .ToString(); - var size = await objectStorage_.AddOrUpdateAsync(resultId, - new List> - { - rc.Data.Memory, - }.ToAsyncEnumerable(), - context.CancellationToken) - .ConfigureAwait(false); - - return new Result(request.SessionId, - resultId, - rc.Name, - "", - request.SessionId, - ResultStatus.Created, - new List(), - DateTime.UtcNow, - size, - Array.Empty()); + var (id, size) = await objectStorage_.AddOrUpdateAsync(new List> + { + rc.Data.Memory, + }.ToAsyncEnumerable(), + context.CancellationToken) + .ConfigureAwait(false); + + return (new Result(request.SessionId, + resultId, + rc.Name, + "", + request.SessionId, + ResultStatus.Created, + new List(), + DateTime.UtcNow, + size, + Array.Empty()), id); }) .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results, + await resultTable_.Create(results.Select(tuple => tuple.Item1) + .AsICollection(), context.CancellationToken) .ConfigureAwait(false); var resultList = await results.Select(async r => await resultTable_.CompleteResult(request.SessionId, - r.ResultId, - r.Size) + r.Item1.ResultId, + r.Item1.Size, + r.id) .ConfigureAwait(false)) .WhenAll() .ConfigureAwait(false); @@ -222,7 +223,14 @@ public override async Task DeleteResultsData(DeleteRe using var measure = meter_.CountAndTime(); try { - await objectStorage_.TryDeleteAsync(request.ResultId, + var opaqueIds = await resultTable_.GetResults(request.SessionId, + request.ResultId, + context.CancellationToken) + .Select(result => result.OpaqueId) + .ToListAsync() + .ConfigureAwait(false); + + await objectStorage_.TryDeleteAsync(opaqueIds, context.CancellationToken) .ConfigureAwait(false); @@ -237,7 +245,14 @@ await objectStorage_.TryDeleteAsync(request.ResultId, catch (ObjectDataNotFoundException e) { logger_.LogWarning(e, - "Error while downloading results"); + "Error while deleting results"); + throw new RpcException(new Status(StatusCode.NotFound, + "Result data not found")); + } + catch (ResultNotFoundException e) + { + logger_.LogWarning(e, + "Error while deleting results"); throw new RpcException(new Status(StatusCode.NotFound, "Result data not found")); } @@ -252,7 +267,10 @@ public override async Task DownloadResultData(DownloadResultDataRequest using var measure = meter_.CountAndTime(); try { - await foreach (var chunk in objectStorage_.GetValuesAsync(request.ResultId, + var id = (await resultTable_.GetResult(request.ResultId) + .ConfigureAwait(false)).OpaqueId; + + await foreach (var chunk in objectStorage_.GetValuesAsync(id, context.CancellationToken) .ConfigureAwait(false)) { @@ -270,6 +288,13 @@ await responseStream.WriteAsync(new DownloadResultDataResponse throw new RpcException(new Status(StatusCode.NotFound, "Result data not found")); } + catch (ResultNotFoundException e) + { + logger_.LogWarning(e, + "Error while downloading results"); + throw new RpcException(new Status(StatusCode.NotFound, + "Result data not found")); + } } [RequiresPermission(typeof(GrpcResultsService), @@ -312,17 +337,11 @@ public override async Task UploadResultData(IAsyncStre var sessionTask = sessionTable_.GetSessionAsync(id.SessionId, context.CancellationToken); - long size = 0; - await objectStorage_.AddOrUpdateAsync(id.ResultId, - requestStream.ReadAllAsync(context.CancellationToken) - .Select(r => - { - size += r.DataChunk.Length; - return r.DataChunk.Memory; - }), - context.CancellationToken) - .ConfigureAwait(false); + var (opaqueId, size) = await objectStorage_.AddOrUpdateAsync(requestStream.ReadAllAsync(context.CancellationToken) + .Select(r => r.DataChunk.Memory), + context.CancellationToken) + .ConfigureAwait(false); try { @@ -331,6 +350,7 @@ await objectStorage_.AddOrUpdateAsync(id.ResultId, var resultData = await resultTable_.CompleteResult(id.SessionId, id.ResultId, size, + opaqueId, context.CancellationToken) .ConfigureAwait(false); diff --git a/Common/src/gRPC/Services/Submitter.cs b/Common/src/gRPC/Services/Submitter.cs index a41d2f575..94380f4ef 100644 --- a/Common/src/gRPC/Services/Submitter.cs +++ b/Common/src/gRPC/Services/Submitter.cs @@ -196,7 +196,7 @@ await responseStream.WriteAsync(new ResultReply } } - await foreach (var chunk in objectStorage_.GetValuesAsync(request.ResultId, + await foreach (var chunk in objectStorage_.GetValuesAsync(result.OpaqueId, cancellationToken) .ConfigureAwait(false)) { @@ -394,15 +394,15 @@ public async Task SetResult(string sessionId, { using var activity = activitySource_.StartActivity($"{nameof(SetResult)}"); - var size = await objectStorage_.AddOrUpdateAsync(key, - chunks, - cancellationToken) - .ConfigureAwait(false); + var (id, size) = await objectStorage_.AddOrUpdateAsync(chunks, + cancellationToken) + .ConfigureAwait(false); await resultTable_.SetResult(sessionId, ownerTaskId, key, size, + id, cancellationToken) .ConfigureAwait(false); } @@ -432,7 +432,7 @@ public async Task> CreateTasks(string ("PartitionId", options.PartitionId)); var requests = new List(); - var payloadUploadTasks = new List>(); + var payloadUploadTasks = new List>(); await foreach (var taskRequest in taskRequests.WithCancellation(cancellationToken) .ConfigureAwait(false)) @@ -447,8 +447,7 @@ public async Task> CreateTasks(string options, taskRequest.ExpectedOutputKeys.ToList(), taskRequest.DataDependencies.ToList())); - payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(payloadId, - taskRequest.PayloadChunks, + payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(taskRequest.PayloadChunks, cancellationToken)); } @@ -457,18 +456,18 @@ public async Task> CreateTasks(string await resultTable_.Create(requests.Zip(payloadSizes, (request, - size) => new Result(sessionId, - request.PayloadId, - "", - parentTaskId.Equals(sessionId) - ? "" - : parentTaskId, - parentTaskId, - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - size, - Array.Empty())) + r) => new Result(sessionId, + request.PayloadId, + "", + parentTaskId.Equals(sessionId) + ? "" + : parentTaskId, + parentTaskId, + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + r.size, + r.id)) .AsICollection(), cancellationToken) .ConfigureAwait(false); @@ -513,12 +512,20 @@ await taskTable_.SetTaskSuccessAsync(taskDataEnd, if (submitterOptions_.DeletePayload) { //Discard value is used to remove warnings CS4014 !! - _ = Task.Factory.StartNew(async () => await objectStorage_.TryDeleteAsync(new[] - { - taskData.PayloadId, - }, - CancellationToken.None) - .ConfigureAwait(false), + _ = Task.Factory.StartNew(async () => + { + var opaqueId = (await resultTable_.GetResult(taskData.PayloadId, + CancellationToken.None) + .ConfigureAwait(false)).OpaqueId; + + + await objectStorage_.TryDeleteAsync(new[] + { + opaqueId, + }, + CancellationToken.None) + .ConfigureAwait(false); + }, cancellationToken); await resultTable_.MarkAsDeleted(taskData.PayloadId, @@ -609,12 +616,20 @@ await taskTable_.SetTaskTimeoutAsync(taskDataEnd, .ConfigureAwait(false); //Discard value is used to remove warnings CS4014 !! - _ = Task.Factory.StartNew(async () => await objectStorage_.TryDeleteAsync(new[] - { - taskData.TaskId, - }, - CancellationToken.None) - .ConfigureAwait(false), + _ = Task.Factory.StartNew(async () => + { + var opaqueId = (await resultTable_.GetResult(taskData.PayloadId, + CancellationToken.None) + .ConfigureAwait(false)).OpaqueId; + + + await objectStorage_.TryDeleteAsync(new[] + { + opaqueId, + }, + CancellationToken.None) + .ConfigureAwait(false); + }, cancellationToken); logger_.LogInformation("Remove input payload of {task}", diff --git a/Common/tests/Helpers/SimpleObjectStorage.cs b/Common/tests/Helpers/SimpleObjectStorage.cs index 677d10d1f..d592cd005 100644 --- a/Common/tests/Helpers/SimpleObjectStorage.cs +++ b/Common/tests/Helpers/SimpleObjectStorage.cs @@ -22,8 +22,8 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -37,19 +37,18 @@ public Task Check(HealthCheckTag tag) public Task Init(CancellationToken cancellationToken) => Task.CompletedTask; - public Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - => Task.FromResult((long)0); + public Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + => Task.FromResult((Encoding.UTF8.GetBytes("id"), (long)0)); - public IAsyncEnumerable GetValuesAsync(string key, + public IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default) => new List { - Encoding.UTF8.GetBytes(key), + id, }.ToAsyncEnumerable(); - - public Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - => Task.CompletedTask; } diff --git a/Common/tests/Helpers/TestDatabaseProvider.cs b/Common/tests/Helpers/TestDatabaseProvider.cs index 0ce3c41c0..e7ed060de 100644 --- a/Common/tests/Helpers/TestDatabaseProvider.cs +++ b/Common/tests/Helpers/TestDatabaseProvider.cs @@ -24,6 +24,7 @@ using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Adapters.MongoDB.Common; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Injection; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Pollster/AgentTest.cs b/Common/tests/Pollster/AgentTest.cs index 8f35e294f..7b89103fa 100644 --- a/Common/tests/Pollster/AgentTest.cs +++ b/Common/tests/Pollster/AgentTest.cs @@ -406,15 +406,6 @@ await holder.Agent.NotifyResultData(holder.Token, CancellationToken.None) .ConfigureAwait(false); - var datAsyncEnumerable = holder.ObjectStorage.GetValuesAsync(ExpectedOutput1, - CancellationToken.None); - - var dataStored = await datAsyncEnumerable.SingleAsync(CancellationToken.None) - .ConfigureAwait(false); - - Assert.AreEqual(data, - dataStored); - await holder.Agent.FinalizeTaskCreation(CancellationToken.None) .ConfigureAwait(false); @@ -431,6 +422,15 @@ await holder.Agent.FinalizeTaskCreation(CancellationToken.None) Assert.AreEqual(data.Length, resultData.Size); + var datAsyncEnumerable = holder.ObjectStorage.GetValuesAsync(resultData.OpaqueId, + CancellationToken.None); + + var dataStored = await datAsyncEnumerable.SingleAsync(CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(data, + dataStored); + var dependents = await holder.ResultTable.GetDependents(holder.Session, ExpectedOutput1, CancellationToken.None) @@ -605,14 +605,28 @@ public async Task GetResourceDataShouldSucceed() { using var holder = new AgentHolder(); - await holder.ObjectStorage.AddOrUpdateAsync("ResourceData", - new List - { - Encoding.ASCII.GetBytes("Data1"), - Encoding.ASCII.GetBytes("Data2"), - }.Select(bytes => new ReadOnlyMemory(bytes)) - .ToAsyncEnumerable(), - CancellationToken.None) + var (id, size) = await holder.ObjectStorage.AddOrUpdateAsync(new List + { + Encoding.ASCII.GetBytes("Data1"), + Encoding.ASCII.GetBytes("Data2"), + }.Select(bytes => new ReadOnlyMemory(bytes)) + .ToAsyncEnumerable(), + CancellationToken.None) + .ConfigureAwait(false); + + await holder.ResultTable.Create(new List + { + new("SessionId", + "ResourceData", + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + size, + id), + }) .ConfigureAwait(false); await holder.Agent.GetResourceData(holder.Token, @@ -657,14 +671,6 @@ public async Task CreateResultsShouldSucceed() resultMetadata.Status); Assert.AreEqual(holder.TaskData.TaskId, resultMetadata.CreatedBy); - - var bytes = (await holder.ObjectStorage.GetValuesAsync(result.ResultId) - .ToListAsync() - .ConfigureAwait(false)).Single(); - - Assert.AreEqual(ByteString.CopyFromUtf8(result.Name) - .ToByteArray(), - bytes); } await holder.Agent.FinalizeTaskCreation(CancellationToken.None) @@ -684,6 +690,14 @@ await holder.Agent.FinalizeTaskCreation(CancellationToken.None) resultMetadata.Status); Assert.AreEqual(7, resultMetadata.Size); + + var bytes = (await holder.ObjectStorage.GetValuesAsync(resultMetadata.OpaqueId) + .ToListAsync() + .ConfigureAwait(false)).Single(); + + Assert.AreEqual(ByteString.CopyFromUtf8(result.Name) + .ToByteArray(), + bytes); } } diff --git a/Common/tests/Pollster/DataPrefetcherTest.cs b/Common/tests/Pollster/DataPrefetcherTest.cs index 492099a1f..450deafaa 100644 --- a/Common/tests/Pollster/DataPrefetcherTest.cs +++ b/Common/tests/Pollster/DataPrefetcherTest.cs @@ -20,12 +20,16 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Linq.Expressions; +using System.Text; using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Common.Tests.Helpers; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -55,10 +59,20 @@ public virtual void TearDown() [Test] public async Task EmptyPayloadAndOneDependency() { + const string sessionId = "SessionId"; + const string parentTaskId = "ParentTaskId"; + const string taskId = "TaskId"; + const string output1 = "Output1"; + const string dependency1 = "Dependency1"; + const string podId = "PodId"; + const string podName = "PodName"; + const string payloadId = "PayloadId"; + const string createdBy = "CreatedBy"; + var mockObjectStorage = new Mock(); - mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), + mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), CancellationToken.None)) - .Returns((string _, + .Returns((byte[] _, CancellationToken _) => new List { Convert.FromBase64String("1111"), @@ -66,21 +80,42 @@ public async Task EmptyPayloadAndOneDependency() Convert.FromBase64String("3333"), Convert.FromBase64String("4444"), }.ToAsyncEnumerable()); + var mockResultTable = new Mock(); + mockResultTable.Setup(x => x.GetResults(It.IsAny>>(), + It.IsAny>>(), + CancellationToken.None)) + .Returns(new List + { + new(sessionId, + dependency1, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(dependency1)), + new(sessionId, + payloadId, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(payloadId)), + }.ToAsyncEnumerable()); + var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + mockResultTable.Object, activitySource_, loggerFactory.CreateLogger()); - const string sessionId = "SessionId"; - const string parentTaskId = "ParentTaskId"; - const string taskId = "TaskId"; - const string output1 = "Output1"; - const string dependency1 = "Dependency1"; - const string podId = "PodId"; - const string podName = "PodName"; - const string payloadId = "PayloadId"; - const string createdBy = "CreatedBy"; + var sharedFolder = Path.Combine(Path.GetTempPath(), "data"); var internalFolder = Path.Combine(Path.GetTempPath(), @@ -135,10 +170,22 @@ await dataPrefetcher.PrefetchDataAsync(new TaskData(sessionId, [Test] public async Task EmptyPayloadAndNoDependenciesStateMachine() { + const string sessionId = "SessionId"; + const string parentTaskId = "ParentTaskId"; + const string taskId = "TaskId"; + const string output1 = "Output1"; + const string dependency1 = "Dependency1"; + const string dependency2 = "Dependency2"; + const string podId = "PodId"; + const string podName = "PodName"; + const string payloadId = "PayloadId"; + const string createdBy = "CreatedBy"; + + var mockObjectStorage = new Mock(); - mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), + mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), CancellationToken.None)) - .Returns((string _, + .Returns((byte[] _, CancellationToken _) => new List { Convert.FromBase64String("1111"), @@ -147,22 +194,52 @@ public async Task EmptyPayloadAndNoDependenciesStateMachine() Convert.FromBase64String("4444"), }.ToAsyncEnumerable()); + var mockResultTable = new Mock(); + mockResultTable.Setup(x => x.GetResults(It.IsAny>>(), + It.IsAny>>(), + CancellationToken.None)) + .Returns(new List + { + new(sessionId, + dependency1, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(dependency1)), + new(sessionId, + dependency2, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(dependency2)), + new(sessionId, + payloadId, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(payloadId)), + }.ToAsyncEnumerable()); + var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + mockResultTable.Object, activitySource_, loggerFactory.CreateLogger()); - const string sessionId = "SessionId"; - const string parentTaskId = "ParentTaskId"; - const string taskId = "TaskId"; - const string output1 = "Output1"; - const string dependency1 = "Dependency1"; - const string dependency2 = "Dependency2"; - const string podId = "PodId"; - const string podName = "PodName"; - const string payloadId = "PayloadId"; - const string createdBy = "CreatedBy"; + var sharedFolder = Path.Combine(Path.GetTempPath(), "data"); var internalFolder = Path.Combine(Path.GetTempPath(), @@ -219,6 +296,7 @@ public async Task InitShouldSucceed() var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new SimpleResultTable(), activitySource_, loggerFactory.CreateLogger()); diff --git a/Common/tests/Pollster/TaskHandlerTest.cs b/Common/tests/Pollster/TaskHandlerTest.cs index 996d01d8c..1291542aa 100644 --- a/Common/tests/Pollster/TaskHandlerTest.cs +++ b/Common/tests/Pollster/TaskHandlerTest.cs @@ -1785,18 +1785,17 @@ public Task Check(HealthCheckTag tag) public Task Init(CancellationToken cancellationToken) => Task.CompletedTask; - public Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - => Task.FromResult(42); - - public IAsyncEnumerable GetValuesAsync(string key, + public IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default) => throw new ObjectDataNotFoundException(); - public Task TryDeleteAsync(IEnumerable keys, + public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + => Task.FromResult<(byte[] id, long size)>((Encoding.UTF8.GetBytes("forty-two"), 42)); } [Test] diff --git a/Common/tests/TaskLifeCycleHelperTest.cs b/Common/tests/TaskLifeCycleHelperTest.cs index 04421b3f5..6535c0fa7 100644 --- a/Common/tests/TaskLifeCycleHelperTest.cs +++ b/Common/tests/TaskLifeCycleHelperTest.cs @@ -427,7 +427,8 @@ await TaskLifeCycleHelper.FinalizeTaskCreation(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[1] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("first data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, @@ -455,7 +456,8 @@ await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[2] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("second data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, @@ -613,12 +615,14 @@ await TaskLifeCycleHelper.FinalizeTaskCreation(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[1] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("first data dependency")) .ConfigureAwait(false); await holder.ResultTable.CompleteResult(sessionId, results[2] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("second data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, holder.ResultTable, diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index fd644238a..12455933f 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -22,9 +22,9 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; -using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -55,25 +55,22 @@ public async Task SetUp() Encoding.ASCII.GetBytes("CCCC"), Encoding.ASCII.GetBytes("DDDD"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey1", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakey1_ = (await ObjectStorage!.AddOrUpdateAsync(dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; dataBytesList = new List> { Encoding.ASCII.GetBytes("AAAABBBB"), }; - await ObjectStorage.AddOrUpdateAsync("dataKey2", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakey2_ = (await ObjectStorage.AddOrUpdateAsync(dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; dataBytesList = new List> { Array.Empty(), }; - await ObjectStorage.AddOrUpdateAsync("dataKeyEmpty", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakeyEmpty_ = (await ObjectStorage.AddOrUpdateAsync(dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; } [TearDown] @@ -94,7 +91,10 @@ private static bool CheckForSkipSetup() /* Boolean to control that tests are executed in * an instance of this class */ - protected bool RunTests; + protected bool RunTests; + private byte[]? datakey1_; + private byte[]? datakey2_; + private byte[]? datakeyEmpty_; /* Function be override so it returns the suitable instance * of TaskTable to the corresponding interface implementation */ @@ -138,11 +138,10 @@ public async Task AddValuesAsyncWithoutChunkShouldWork() { if (RunTests) { - var size = await ObjectStorage!.AddOrUpdateAsync("dataKeyNoChunk", - AsyncEnumerable.Empty>()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(AsyncEnumerable.Empty>()) + .ConfigureAwait(false); var data = new List(); - await foreach (var chunk in ObjectStorage!.GetValuesAsync("dataKeyNoChunk") + await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) .ConfigureAwait(false)) { data.AddRange(chunk); @@ -174,13 +173,12 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) { if (RunTests) { - var size = await ObjectStorage!.AddOrUpdateAsync("dataKeyTest", - inputChunks.ToAsyncEnumerable() - .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(inputChunks.ToAsyncEnumerable() + .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) + .ConfigureAwait(false); var data = new List(); - await foreach (var chunk in ObjectStorage!.GetValuesAsync("dataKeyTest") + await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) .ConfigureAwait(false)) { data.AddRange(chunk); @@ -201,7 +199,8 @@ public void GetValuesAsyncShouldFail() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKeyNotExist"); + var res = ObjectStorage!.GetValuesAsync(Guid.NewGuid() + .ToByteArray()); Assert.ThrowsAsync(async () => await res.FirstAsync() .ConfigureAwait(false)); } @@ -212,7 +211,7 @@ public async Task PayloadShouldBeEqual() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKey2"); + var res = ObjectStorage!.GetValuesAsync(datakey2_!); var data = new List(); foreach (var item in await res.ToListAsync() .ConfigureAwait(false)) @@ -231,7 +230,7 @@ public async Task Payload2ShouldBeEqual() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKey1"); + var res = ObjectStorage!.GetValuesAsync(datakey1_!); // var data = await res.AggregateAsync((bytes1, bytes2) => bytes1.Concat(bytes2).ToArray()); var data = new List(); foreach (var item in await res.ToListAsync() @@ -251,7 +250,7 @@ public async Task EmptyPayload() { if (RunTests) { - var res = await ObjectStorage!.GetValuesAsync("dataKeyEmpty") + var res = await ObjectStorage!.GetValuesAsync(datakeyEmpty_!) .ToListAsync() .ConfigureAwait(false); Console.WriteLine(res.Count); @@ -281,11 +280,10 @@ public async Task DeleteKeysAndGetValuesAsyncShouldFail() Encoding.ASCII.GetBytes("Data 4"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey", - listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(listChunks.ToAsyncEnumerable()) + .ConfigureAwait(false); - var res = await ObjectStorage!.GetValuesAsync("dataKey") + var res = await ObjectStorage!.GetValuesAsync(id) .ToListAsync() .ConfigureAwait(false); Assert.AreEqual(string.Join("", @@ -295,11 +293,11 @@ public async Task DeleteKeysAndGetValuesAsyncShouldFail() await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); - Assert.ThrowsAsync(async () => await ObjectStorage!.GetValuesAsync("dataKey") + Assert.ThrowsAsync(async () => await ObjectStorage!.GetValuesAsync(id) .FirstAsync() .ConfigureAwait(false)); } @@ -319,19 +317,18 @@ public async Task DeleteDeleteTwiceShouldSucceed() Encoding.ASCII.GetBytes("Data 4"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey", - listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(listChunks.ToAsyncEnumerable()) + .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); } diff --git a/Common/tests/TestBase/ResultTableTestBase.cs b/Common/tests/TestBase/ResultTableTestBase.cs index 01381592e..1c39e1068 100644 --- a/Common/tests/TestBase/ResultTableTestBase.cs +++ b/Common/tests/TestBase/ResultTableTestBase.cs @@ -324,48 +324,25 @@ public async Task SetResultShouldSucceed() { if (RunTests) { - await ResultTable!.SetResult((string)"SessionId", - (string)"OwnerId", - (string)"ResultIsNotAvailable", - 5, - CancellationToken.None) - .ConfigureAwait(false); - - var result = await ResultTable!.GetResult("ResultIsNotAvailable", - CancellationToken.None) - .ConfigureAwait(false); - - Assert.IsTrue(result.ResultId == "ResultIsNotAvailable"); - Assert.AreEqual(5, - result.Size); - } - } - - [Test] - public async Task SetResultSmallPayloadShouldSucceed() - { - if (RunTests) - { - var smallPayload = new[] - { - (byte)1, - (byte)2, - }; - + var id = Encoding.UTF8.GetBytes("OpaqueId"); await ResultTable!.SetResult("SessionId", "OwnerId", "ResultIsNotAvailable", - smallPayload, + 5, + id, CancellationToken.None) .ConfigureAwait(false); + var result = await ResultTable!.GetResult("ResultIsNotAvailable", CancellationToken.None) .ConfigureAwait(false); - Assert.AreEqual(result.Data, - smallPayload); - Assert.AreEqual(smallPayload.Length, + Assert.AreEqual("ResultIsNotAvailable", + result.ResultId); + Assert.AreEqual(5, result.Size); + Assert.AreEqual(id, + result.OpaqueId); } } @@ -770,6 +747,7 @@ public async Task CompleteResultShouldSucceed() .ToString(); var sessionId = Guid.NewGuid() .ToString(); + var id = Encoding.UTF8.GetBytes("OpaqueId"); await ResultTable!.Create(new List { new(sessionId, @@ -789,11 +767,16 @@ public async Task CompleteResultShouldSucceed() var result = await ResultTable.CompleteResult(sessionId, resultId, 5, + id, CancellationToken.None) .ConfigureAwait(false); Assert.AreEqual(ResultStatus.Completed, result.Status); + Assert.AreEqual(id, + result.OpaqueId); + Assert.AreEqual(5, + result.Size); result = await ResultTable.GetResult(resultId, CancellationToken.None) @@ -803,6 +786,8 @@ public async Task CompleteResultShouldSucceed() result.Status); Assert.AreEqual(5, result.Size); + Assert.AreEqual(id, + result.OpaqueId); } } @@ -811,9 +796,11 @@ public void CompleteResultShouldThrow() { if (RunTests) { + var id = Encoding.UTF8.GetBytes("OpaqueId"); Assert.ThrowsAsync(async () => await ResultTable!.CompleteResult("SessionId", "NotExistingResult111", 5, + id, CancellationToken.None) .ConfigureAwait(false)); } From 0574c28670c74c344e9210d3e0c25859ed4c6a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 7 Nov 2024 09:50:28 +0100 Subject: [PATCH 05/15] feat: add embedded object storage that stores object data into the database --- .github/workflows/build.yml | 11 ++- .../src/ArmoniK.Core.Adapters.Embed.csproj | 75 ++++++++++++++++ ...niK.Core.Adapters.Embed.csproj.DotSettings | 2 + Adaptors/Embed/src/ObjectBuilder.cs | 41 +++++++++ Adaptors/Embed/src/ObjectStorage.cs | 86 +++++++++++++++++++ .../ArmoniK.Core.Adapters.Embed.Tests.csproj | 28 ++++++ ...re.Adapters.Embed.Tests.csproj.DotSettings | 2 + Adaptors/Embed/tests/ObjectStorageTests.cs | 78 +++++++++++++++++ ArmoniK.Core.sln | 22 +++++ .../tests/TestBase/ObjectStorageTestBase.cs | 67 ++++++--------- Dockerfile | 10 +++ justfile | 3 + terraform/locals.tf | 2 +- terraform/main.tf | 5 ++ .../modules/storage/object/embed/output.tf | 11 +++ terraform/variables.tf | 4 +- 16 files changed, 396 insertions(+), 51 deletions(-) create mode 100644 Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj create mode 100644 Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings create mode 100644 Adaptors/Embed/src/ObjectBuilder.cs create mode 100644 Adaptors/Embed/src/ObjectStorage.cs create mode 100644 Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj create mode 100644 Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings create mode 100644 Adaptors/Embed/tests/ObjectStorageTests.cs create mode 100644 terraform/modules/storage/object/embed/output.tf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ccd4a441..cddd264ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,6 +40,7 @@ jobs: - Adaptors/MongoDB/tests - Adaptors/Memory/tests - Adaptors/S3/tests + - Adaptors/Embed/tests os: - ubuntu-latest fail-fast: false @@ -179,6 +180,7 @@ jobs: - Common/tests - Adaptors/MongoDB/tests - Adaptors/Memory/tests + - Adaptors/Embed/tests fail-fast: false runs-on: windows-latest steps: @@ -189,14 +191,10 @@ jobs: submodules: true - name: Dotnet Restore - run: | - cd ${{ matrix.projects }} - dotnet restore + run: dotnet restore - name: Dotnet Build - run: | - cd ${{ matrix.projects }} - dotnet build + run: dotnet build - name: Run tests run: | @@ -442,6 +440,7 @@ jobs: object: - redis - minio + - embed log-level: - Information - Verbose diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj new file mode 100644 index 000000000..da557fc65 --- /dev/null +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj @@ -0,0 +1,75 @@ + + + net8.0 + win-x64;linux-x64;linux-arm64 + True + ANEO + Copyright (C) ANEO, 2021-2022 + AGPL-3.0-or-later + True + true + enable + true + + + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + + + + diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/Adaptors/Embed/src/ObjectBuilder.cs b/Adaptors/Embed/src/ObjectBuilder.cs new file mode 100644 index 000000000..cd41e8fc0 --- /dev/null +++ b/Adaptors/Embed/src/ObjectBuilder.cs @@ -0,0 +1,41 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.Embed; + +/// +/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + => serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); +} diff --git a/Adaptors/Embed/src/ObjectStorage.cs b/Adaptors/Embed/src/ObjectStorage.cs new file mode 100644 index 000000000..2eeced886 --- /dev/null +++ b/Adaptors/Embed/src/ObjectStorage.cs @@ -0,0 +1,86 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.Embed; + +[UsedImplicitly] +public class ObjectStorage : IObjectStorage +{ + private readonly ILogger logger_; + private bool isInitialized_; + + /// + /// implementation for Redis + /// + /// Logger used to print logs + public ObjectStorage(ILogger logger) + => logger_ = logger; + + /// + public Task Init(CancellationToken cancellationToken) + { + isInitialized_ = true; + return Task.CompletedTask; + } + + /// + public Task Check(HealthCheckTag tag) + => Task.FromResult(isInitialized_ + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy("Object storage not initialized yet.")); + + /// + public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + { + var array = new List(); + + await foreach (var val in valueChunks.WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + array.AddRange(val.ToArray()); + } + + return (array.ToArray(), array.Count); + } + + /// + public async IAsyncEnumerable GetValuesAsync(byte[] id, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + yield return id; + } + + /// + public Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => Task.CompletedTask; +} diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj new file mode 100644 index 000000000..975cca170 --- /dev/null +++ b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj @@ -0,0 +1,28 @@ + + + net8.0 + win-x64;linux-x64;linux-arm64 + false + enable + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/Adaptors/Embed/tests/ObjectStorageTests.cs b/Adaptors/Embed/tests/ObjectStorageTests.cs new file mode 100644 index 000000000..472d36a4a --- /dev/null +++ b/Adaptors/Embed/tests/ObjectStorageTests.cs @@ -0,0 +1,78 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System.Collections.Generic; +using System.IO; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; +using ArmoniK.Core.Common.Tests.TestBase; +using ArmoniK.Core.Common.Utils; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +using NUnit.Framework; + +namespace ArmoniK.Core.Adapters.Embed.Tests; + +[TestFixture] +public class ObjectStorageTests : ObjectStorageTestBase +{ + private static readonly string SolutionRoot = + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests) + .Assembly + .Location))))) ?? + string.Empty)); + + private static readonly string S3Path = + $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.Embed.dll"; + + + protected override void GetObjectStorageInstance() + { + Dictionary minimalConfig = new() + { + { + "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.Embed.ObjectBuilder" + }, + { + "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}" + }, + }; + + var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(minimalConfig); + + var services = new ServiceCollection(); + services.AddLogging(); + var logger = new LoggerInit(configuration); + + services.AddAdapter(configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()); + + var provider = services.BuildServiceProvider(new ServiceProviderOptions + { + ValidateOnBuild = true, + }); + + ObjectStorage = provider.GetRequiredService(); + RunTests = true; + } +} diff --git a/ArmoniK.Core.sln b/ArmoniK.Core.sln index 05b8c8138..fb1b1af96 100644 --- a/ArmoniK.Core.sln +++ b/ArmoniK.Core.sln @@ -101,6 +101,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Tests.Connecti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Adapters.SQS", "Adaptors\SQS\src\ArmoniK.Core.Adapters.SQS.csproj", "{399C779C-CE8D-4757-8098-7F055467D96C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.Adapters.Embed", "Adaptors\Embed\src\ArmoniK.Core.Adapters.Embed.csproj", "{F95A1BF7-B660-4789-9F2D-1749FC104EEE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.Adapters.Embed.Tests", "Adaptors\Embed\tests\ArmoniK.Core.Adapters.Embed.Tests.csproj", "{2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -413,6 +417,22 @@ Global {399C779C-CE8D-4757-8098-7F055467D96C}.Release|Any CPU.Build.0 = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.ActiveCfg = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.Build.0 = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|x64.ActiveCfg = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|x64.Build.0 = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|Any CPU.Build.0 = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|x64.ActiveCfg = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|x64.Build.0 = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|x64.Build.0 = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|Any CPU.Build.0 = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|x64.ActiveCfg = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -465,6 +485,8 @@ Global {9F19E6DD-D9CD-43EC-B5CC-0081997ED8B8} = {6EE499A5-760F-4823-8AB6-DBDE8C5B5F45} {6ABF29BB-2F42-4088-AE93-E41416CB3271} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} {399C779C-CE8D-4757-8098-7F055467D96C} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {F95A1BF7-B660-4789-9F2D-1749FC104EEE} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E} = {2D548C40-6260-4A4E-9C56-E8A80C639E13} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1285A466-2AF6-43E6-8DCC-2F93A5D5F02E} diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 12455933f..2b556d5bb 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -195,14 +195,34 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) } [Test] - public void GetValuesAsyncShouldFail() + public async Task GetValuesAsyncShouldFail() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync(Guid.NewGuid() - .ToByteArray()); - Assert.ThrowsAsync(async () => await res.FirstAsync() - .ConfigureAwait(false)); + var id = Guid.NewGuid() + .ToByteArray(); + var data = new List(); + + try + { + var res = ObjectStorage!.GetValuesAsync(id); + + + await foreach (var chunk in res.ConfigureAwait(false)) + { + data.AddRange(chunk); + } + + Assert.AreEqual(id, + data.ToArray()); + } + catch (ObjectDataNotFoundException) + { + } + catch (Exception) + { + Assert.Fail("When value should not be available, it should either throw or return the value"); + } } } @@ -266,43 +286,6 @@ public async Task EmptyPayload() } } - [Test] - public async Task DeleteKeysAndGetValuesAsyncShouldFail() - { - if (RunTests) - { - var listChunks = new List> - { - Encoding.ASCII.GetBytes("Armonik Payload chunk"), - Encoding.ASCII.GetBytes("Data 1"), - Encoding.ASCII.GetBytes("Data 2"), - Encoding.ASCII.GetBytes("Data 3"), - Encoding.ASCII.GetBytes("Data 4"), - }; - - var (id, size) = await ObjectStorage!.AddOrUpdateAsync(listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); - - var res = await ObjectStorage!.GetValuesAsync(id) - .ToListAsync() - .ConfigureAwait(false); - Assert.AreEqual(string.Join("", - listChunks.Select(chunk => Encoding.ASCII.GetString(chunk.ToArray()))), - string.Join("", - res.Select(chunk => Encoding.ASCII.GetString(chunk)))); - - await ObjectStorage!.TryDeleteAsync(new[] - { - id, - }) - .ConfigureAwait(false); - - Assert.ThrowsAsync(async () => await ObjectStorage!.GetValuesAsync(id) - .FirstAsync() - .ConfigureAwait(false)); - } - } - [Test] public async Task DeleteDeleteTwiceShouldSucceed() { diff --git a/Dockerfile b/Dockerfile index 02a57a11b..780b0e63f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY ["Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj", "Adaptors/PubSu COPY ["Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj", "Adaptors/Redis/src/"] COPY ["Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj", "Adaptors/S3/src/"] COPY ["Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj", "Adaptors/SQS/src/"] +COPY ["Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj", "Adaptors/Embed/src/"] COPY ["Base/src/ArmoniK.Core.Base.csproj", "Base/src/"] COPY ["Common/src/ArmoniK.Core.Common.csproj", "Common/src/"] COPY ["Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj", "Compute/PollingAgent/src/"] @@ -43,6 +44,7 @@ RUN dotnet restore -a "${TARGETARCH}" "Adaptors/SQS/src/ArmoniK.Core.Adapters.SQ RUN dotnet restore -a "${TARGETARCH}" "Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj" # git ls-tree -r HEAD --name-only --full-tree | grep "csproj$" | xargs -I % sh -c "export D=\$(dirname %) ; echo COPY [\\\"\$D\\\", \\\"\$D\\\"]" COPY ["Adaptors/Amqp/src", "Adaptors/Amqp/src"] @@ -55,6 +57,7 @@ COPY ["Adaptors/PubSub/src", "Adaptors/PubSub/src"] COPY ["Adaptors/Redis/src", "Adaptors/Redis/src"] COPY ["Adaptors/S3/src", "Adaptors/S3/src"] COPY ["Adaptors/SQS/src", "Adaptors/SQS/src"] +COPY ["Adaptors/Embed/src", "Adaptors/Embed/src"] COPY ["Base/src", "Base/src"] COPY ["Common/src", "Common/src"] COPY ["Compute/PollingAgent/src", "Compute/PollingAgent/src"] @@ -78,6 +81,9 @@ RUN dotnet publish "ArmoniK.Core.Adapters.RabbitMQ.csproj" -a "${TARGETARCH}" -- WORKDIR /src/Adaptors/LocalStorage/src RUN dotnet publish "ArmoniK.Core.Adapters.LocalStorage.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/local_storage /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION +WORKDIR /src/Adaptors/Embed/src +RUN dotnet publish "ArmoniK.Core.Adapters.Embed.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/embed /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + WORKDIR /src/Adaptors/Redis/src RUN dotnet publish "ArmoniK.Core.Adapters.Redis.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/redis /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION @@ -110,6 +116,8 @@ WORKDIR /adapters/object/local_storage COPY --from=build /app/publish/local_storage . WORKDIR /adapters/object/redis COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/embed +COPY --from=build /app/publish/embed . WORKDIR /adapters/object/s3 COPY --from=build /app/publish/s3 . WORKDIR /app @@ -148,6 +156,8 @@ WORKDIR /adapters/object/local_storage COPY --from=build /app/publish/local_storage . WORKDIR /adapters/object/redis COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/embed +COPY --from=build /app/publish/embed . WORKDIR /adapters/object/s3 COPY --from=build /app/publish/s3 . WORKDIR /app diff --git a/justfile b/justfile index 8b255cc74..dad19ad66 100644 --- a/justfile +++ b/justfile @@ -62,6 +62,8 @@ object_storage := if object == "redis" { '{ name = "redis", image = "redis:bullseye" ' } else if object == "minio" { '{ name = "minio", image = "quay.io/minio/minio" ' +} else if object == "embed" { + '{ name = "embed"' } else { '{ name = "local", image = "" ' } @@ -179,6 +181,7 @@ _usage: redis: to use redis for object storage (default) minio: to use minio for object storage. local: to mount a local volume for object storage + embed: to use the database as an object storage replicas: Number of polling agents / worker to be replicated (default = 3) diff --git a/terraform/locals.tf b/terraform/locals.tf index e5e81099e..074743fc6 100644 --- a/terraform/locals.tf +++ b/terraform/locals.tf @@ -16,7 +16,7 @@ locals { worker = merge(var.compute_plane.worker, { image = var.worker_image }) queue = one(concat(module.queue_activemq, module.queue_rabbitmq, module.queue_artemis, module.queue_pubsub, module.queue_sqs, module.queue_none)) database = module.database - object = one(concat(module.object_redis, module.object_minio, module.object_local)) + object = one(concat(module.object_redis, module.object_minio, module.object_local, module.object_embed)) env_maps = concat([ local.queue.generated_env_vars, local.object.generated_env_vars, diff --git a/terraform/main.tf b/terraform/main.tf index 56f9b4b2b..b20d0b968 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -60,6 +60,11 @@ module "object_local" { local_path = var.object_storage.local_storage_path } +module "object_embed" { + source = "./modules/storage/object/embed" + count = var.object_storage.name == "embed" ? 1 : 0 +} + module "queue_rabbitmq" { source = "./modules/storage/queue/rabbitmq" count = var.queue_storage.name == "rabbitmq" ? 1 : 0 diff --git a/terraform/modules/storage/object/embed/output.tf b/terraform/modules/storage/object/embed/output.tf new file mode 100644 index 000000000..3109defe1 --- /dev/null +++ b/terraform/modules/storage/object/embed/output.tf @@ -0,0 +1,11 @@ +output "generated_env_vars" { + value = ({ + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Embed.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/embed/ArmoniK.Core.Adapters.Embed.dll" + }) +} + +output "volumes" { + description = "Volumes that agents and submitters must mount to access the object storage" + value = {} +} diff --git a/terraform/variables.tf b/terraform/variables.tf index ea1eb649a..23efed83c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -62,8 +62,8 @@ variable "object_storage" { local_storage_path = optional(string, "/local_storage") }) validation { - condition = can(regex("^(redis|local|minio)$", var.object_storage.name)) - error_message = "Must be redis, minio, or local" + condition = can(regex("^(redis|local|minio|embed)$", var.object_storage.name)) + error_message = "Must be redis, minio, embed, or local" } default = {} } From e58715705a89fd5d14e0e000467f6820fc65afcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Fri, 15 Nov 2024 16:18:05 +0100 Subject: [PATCH 06/15] refactor: use projections --- Common/src/Pollster/Agent.cs | 41 +++++++++---------- Common/src/Pollster/DataPrefetcher.cs | 21 +++++----- .../src/gRPC/Services/GrpcResultsService.cs | 7 ++-- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index b0cab6aae..6d4ee7486 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -265,12 +265,19 @@ public async Task> CreateResults(string { ThrowIfInvalidToken(token); - var local = new Dictionary(); - var results = await requests.Select(async rc => { - var resultId = Guid.NewGuid() - .ToString(); + var result = new Result(rc.request.SessionId, + Guid.NewGuid() + .ToString(), + rc.request.Name, + taskData_.TaskId, + "", + ResultStatus.Created, + new List(), + DateTime.UtcNow, + 0, + Array.Empty()); var add = await objectStorage_.AddOrUpdateAsync(new List> { @@ -279,34 +286,24 @@ public async Task> CreateResults(string cancellationToken) .ConfigureAwait(false); - local.Add(resultId, - add); - - return new Result(rc.request.SessionId, - resultId, - rc.request.Name, - taskData_.TaskId, - "", - ResultStatus.Created, - new List(), - DateTime.UtcNow, - 0, - Array.Empty()); + return (result, add); }) .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results, + await resultTable_.Create(results.Select(tuple => tuple.result) + .AsICollection(), cancellationToken) .ConfigureAwait(false); - foreach (var result in local) + foreach (var result in results) { - sentResults_.Add(result.Key, - result.Value); + sentResults_.Add(result.result.ResultId, + result.add); } - return results; + return results.Select(tuple => tuple.result) + .AsICollection(); } /// diff --git a/Common/src/Pollster/DataPrefetcher.cs b/Common/src/Pollster/DataPrefetcher.cs index a6c658589..ae566f9f6 100644 --- a/Common/src/Pollster/DataPrefetcher.cs +++ b/Common/src/Pollster/DataPrefetcher.cs @@ -18,7 +18,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -106,20 +105,20 @@ public async Task PrefetchDataAsync(TaskData taskData, dependencies.AddRange(taskData.DataDependencies); - var ids = await resultTable_.GetResults(taskData.SessionId, - dependencies, - cancellationToken) - .Select(r => (r.ResultId, r.OpaqueId)) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - foreach (var (resultId, opaqueId) in ids) + await foreach (var id in resultTable_.GetResults(data => dependencies.Contains(data.ResultId), + r => new + { + r.ResultId, + r.OpaqueId, + }, + cancellationToken) + .ConfigureAwait(false)) { await using var fs = new FileStream(Path.Combine(folder, - resultId), + id.ResultId), FileMode.OpenOrCreate); await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(opaqueId, + await foreach (var chunk in objectStorage_.GetValuesAsync(id.OpaqueId, cancellationToken) .ConfigureAwait(false)) { diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index 1c4e7ac56..85504e99d 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -223,11 +223,10 @@ public override async Task DeleteResultsData(DeleteRe using var measure = meter_.CountAndTime(); try { - var opaqueIds = await resultTable_.GetResults(request.SessionId, - request.ResultId, + var opaqueIds = await resultTable_.GetResults(result => request.ResultId.Contains(result.ResultId), + result => result.OpaqueId, context.CancellationToken) - .Select(result => result.OpaqueId) - .ToListAsync() + .ToListAsync(context.CancellationToken) .ConfigureAwait(false); await objectStorage_.TryDeleteAsync(opaqueIds, From 72a9c6fa40af1e1c58bb39981bfe9120e1716183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Fri, 15 Nov 2024 17:08:01 +0100 Subject: [PATCH 07/15] feat: pass metadata to the object storage --- Adaptors/Embed/src/ObjectStorage.cs | 3 +- Adaptors/LocalStorage/src/ObjectStorage.cs | 3 +- Adaptors/Memory/src/ObjectStorage.cs | 3 +- Adaptors/Redis/src/ObjectStorage.cs | 3 +- Adaptors/S3/src/ObjectStorage.cs | 3 +- Base/src/DataStructures/ObjectData.cs | 29 +++++++++++++++ Base/src/IObjectStorage.cs | 4 ++- Common/src/Pollster/Agent.cs | 13 +++++-- .../src/gRPC/Services/GrpcResultsService.cs | 13 +++++-- Common/src/gRPC/Services/Submitter.cs | 13 +++++-- Common/tests/Helpers/SimpleObjectStorage.cs | 3 +- Common/tests/Pollster/AgentTest.cs | 14 +++++--- Common/tests/Pollster/TaskHandlerTest.cs | 3 +- .../tests/TestBase/ObjectStorageTestBase.cs | 36 +++++++++++++++---- 14 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 Base/src/DataStructures/ObjectData.cs diff --git a/Adaptors/Embed/src/ObjectStorage.cs b/Adaptors/Embed/src/ObjectStorage.cs index 2eeced886..bc4ea87b5 100644 --- a/Adaptors/Embed/src/ObjectStorage.cs +++ b/Adaptors/Embed/src/ObjectStorage.cs @@ -58,7 +58,8 @@ public Task Check(HealthCheckTag tag) : HealthCheckResult.Unhealthy("Object storage not initialized yet.")); /// - public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { var array = new List(); diff --git a/Adaptors/LocalStorage/src/ObjectStorage.cs b/Adaptors/LocalStorage/src/ObjectStorage.cs index ea8e44174..2f9d99ac6 100644 --- a/Adaptors/LocalStorage/src/ObjectStorage.cs +++ b/Adaptors/LocalStorage/src/ObjectStorage.cs @@ -93,7 +93,8 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { var id = Guid.NewGuid(); diff --git a/Adaptors/Memory/src/ObjectStorage.cs b/Adaptors/Memory/src/ObjectStorage.cs index b9dd2df41..2495ef11f 100644 --- a/Adaptors/Memory/src/ObjectStorage.cs +++ b/Adaptors/Memory/src/ObjectStorage.cs @@ -49,7 +49,8 @@ public Task Check(HealthCheckTag tag) ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy()); - public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { var array = new List(); diff --git a/Adaptors/Redis/src/ObjectStorage.cs b/Adaptors/Redis/src/ObjectStorage.cs index 010db5852..e80c27f6d 100644 --- a/Adaptors/Redis/src/ObjectStorage.cs +++ b/Adaptors/Redis/src/ObjectStorage.cs @@ -86,7 +86,8 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { var id = Guid.NewGuid(); diff --git a/Adaptors/S3/src/ObjectStorage.cs b/Adaptors/S3/src/ObjectStorage.cs index 572502aaf..939a2bd4c 100644 --- a/Adaptors/S3/src/ObjectStorage.cs +++ b/Adaptors/S3/src/ObjectStorage.cs @@ -164,7 +164,8 @@ await s3Client_.GetObjectAsync(options_.BucketName, } /// - public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { diff --git a/Base/src/DataStructures/ObjectData.cs b/Base/src/DataStructures/ObjectData.cs new file mode 100644 index 000000000..73ed37078 --- /dev/null +++ b/Base/src/DataStructures/ObjectData.cs @@ -0,0 +1,29 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +namespace ArmoniK.Core.Base.DataStructures; + +/// +/// Metadata given to the +/// +public record ObjectData +{ + /// + /// Id of the result + /// + public required string ResultId; +} diff --git a/Base/src/IObjectStorage.cs b/Base/src/IObjectStorage.cs index b1bd25a51..13257773f 100644 --- a/Base/src/IObjectStorage.cs +++ b/Base/src/IObjectStorage.cs @@ -20,6 +20,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; namespace ArmoniK.Core.Base; @@ -43,7 +44,8 @@ public interface IObjectStorage : IInitializable /// Opaque ID will be used to refer to the object. The implementation of this interface should generate it. /// /// the key is not found - Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData data, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default); /// diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index 6d4ee7486..2c57918bd 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Api.gRPC.V1; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; @@ -279,7 +280,11 @@ public async Task> CreateResults(string 0, Array.Empty()); - var add = await objectStorage_.AddOrUpdateAsync(new List> + var add = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = result.ResultId, + }, + new List> { rc.data, }.ToAsyncEnumerable(), @@ -321,7 +326,11 @@ public async Task> NotifyResultData(string toke using var r = new BinaryReader(fs); var channel = Channel.CreateUnbounded>(); - var addTask = objectStorage_.AddOrUpdateAsync(channel.Reader.ReadAllAsync(cancellationToken), + var addTask = objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = result, + }, + channel.Reader.ReadAllAsync(cancellationToken), cancellationToken); int read; diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index 85504e99d..5a815e1bf 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1.Results; using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; @@ -172,7 +173,11 @@ public override async Task CreateResults(CreateResultsReq var resultId = Guid.NewGuid() .ToString(); - var (id, size) = await objectStorage_.AddOrUpdateAsync(new List> + var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = resultId, + }, + new List> { rc.Data.Memory, }.ToAsyncEnumerable(), @@ -337,7 +342,11 @@ public override async Task UploadResultData(IAsyncStre context.CancellationToken); - var (opaqueId, size) = await objectStorage_.AddOrUpdateAsync(requestStream.ReadAllAsync(context.CancellationToken) + var (opaqueId, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = id.ResultId, + }, + requestStream.ReadAllAsync(context.CancellationToken) .Select(r => r.DataChunk.Memory), context.CancellationToken) .ConfigureAwait(false); diff --git a/Common/src/gRPC/Services/Submitter.cs b/Common/src/gRPC/Services/Submitter.cs index 94380f4ef..10804c106 100644 --- a/Common/src/gRPC/Services/Submitter.cs +++ b/Common/src/gRPC/Services/Submitter.cs @@ -27,6 +27,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Storage; @@ -394,7 +395,11 @@ public async Task SetResult(string sessionId, { using var activity = activitySource_.StartActivity($"{nameof(SetResult)}"); - var (id, size) = await objectStorage_.AddOrUpdateAsync(chunks, + var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = key, + }, + chunks, cancellationToken) .ConfigureAwait(false); @@ -447,7 +452,11 @@ public async Task> CreateTasks(string options, taskRequest.ExpectedOutputKeys.ToList(), taskRequest.DataDependencies.ToList())); - payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(taskRequest.PayloadChunks, + payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = payloadId, + }, + taskRequest.PayloadChunks, cancellationToken)); } diff --git a/Common/tests/Helpers/SimpleObjectStorage.cs b/Common/tests/Helpers/SimpleObjectStorage.cs index d592cd005..782f269cd 100644 --- a/Common/tests/Helpers/SimpleObjectStorage.cs +++ b/Common/tests/Helpers/SimpleObjectStorage.cs @@ -41,7 +41,8 @@ public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) => Task.FromResult((Encoding.UTF8.GetBytes("id"), (long)0)); diff --git a/Common/tests/Pollster/AgentTest.cs b/Common/tests/Pollster/AgentTest.cs index 7b89103fa..09162b52c 100644 --- a/Common/tests/Pollster/AgentTest.cs +++ b/Common/tests/Pollster/AgentTest.cs @@ -605,12 +605,16 @@ public async Task GetResourceDataShouldSucceed() { using var holder = new AgentHolder(); - var (id, size) = await holder.ObjectStorage.AddOrUpdateAsync(new List + var (id, size) = await holder.ObjectStorage.AddOrUpdateAsync(new ObjectData { - Encoding.ASCII.GetBytes("Data1"), - Encoding.ASCII.GetBytes("Data2"), - }.Select(bytes => new ReadOnlyMemory(bytes)) - .ToAsyncEnumerable(), + ResultId = "", + }, + new List + { + Encoding.ASCII.GetBytes("Data1"), + Encoding.ASCII.GetBytes("Data2"), + }.Select(bytes => new ReadOnlyMemory(bytes)) + .ToAsyncEnumerable(), CancellationToken.None) .ConfigureAwait(false); diff --git a/Common/tests/Pollster/TaskHandlerTest.cs b/Common/tests/Pollster/TaskHandlerTest.cs index 1291542aa..093f308d0 100644 --- a/Common/tests/Pollster/TaskHandlerTest.cs +++ b/Common/tests/Pollster/TaskHandlerTest.cs @@ -1793,7 +1793,8 @@ public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable> valueChunks, + public Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) => Task.FromResult<(byte[] id, long size)>((Encoding.UTF8.GetBytes("forty-two"), 42)); } diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 2b556d5bb..0fc8bc544 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -55,21 +55,33 @@ public async Task SetUp() Encoding.ASCII.GetBytes("CCCC"), Encoding.ASCII.GetBytes("DDDD"), }; - datakey1_ = (await ObjectStorage!.AddOrUpdateAsync(dataBytesList.ToAsyncEnumerable()) + datakey1_ = (await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = string.Empty, + }, + dataBytesList.ToAsyncEnumerable()) .ConfigureAwait(false)).id; dataBytesList = new List> { Encoding.ASCII.GetBytes("AAAABBBB"), }; - datakey2_ = (await ObjectStorage.AddOrUpdateAsync(dataBytesList.ToAsyncEnumerable()) + datakey2_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = string.Empty, + }, + dataBytesList.ToAsyncEnumerable()) .ConfigureAwait(false)).id; dataBytesList = new List> { Array.Empty(), }; - datakeyEmpty_ = (await ObjectStorage.AddOrUpdateAsync(dataBytesList.ToAsyncEnumerable()) + datakeyEmpty_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = string.Empty, + }, + dataBytesList.ToAsyncEnumerable()) .ConfigureAwait(false)).id; } @@ -138,7 +150,11 @@ public async Task AddValuesAsyncWithoutChunkShouldWork() { if (RunTests) { - var (id, size) = await ObjectStorage!.AddOrUpdateAsync(AsyncEnumerable.Empty>()) + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = string.Empty, + }, + AsyncEnumerable.Empty>()) .ConfigureAwait(false); var data = new List(); await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) @@ -173,7 +189,11 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) { if (RunTests) { - var (id, size) = await ObjectStorage!.AddOrUpdateAsync(inputChunks.ToAsyncEnumerable() + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = string.Empty, + }, + inputChunks.ToAsyncEnumerable() .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) .ConfigureAwait(false); @@ -300,7 +320,11 @@ public async Task DeleteDeleteTwiceShouldSucceed() Encoding.ASCII.GetBytes("Data 4"), }; - var (id, size) = await ObjectStorage!.AddOrUpdateAsync(listChunks.ToAsyncEnumerable()) + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = string.Empty, + }, + listChunks.ToAsyncEnumerable()) .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] From 43a0976b91c4b497b02d9da6098c60bd7da6bb0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 5 Dec 2024 08:17:51 +0100 Subject: [PATCH 08/15] feat: add SessionId in ObjectData --- Base/src/DataStructures/ObjectData.cs | 5 +++++ Common/src/Pollster/Agent.cs | 6 ++++-- Common/src/gRPC/Services/GrpcResultsService.cs | 6 ++++-- Common/src/gRPC/Services/Submitter.cs | 6 ++++-- Common/tests/Pollster/AgentTest.cs | 3 ++- Common/tests/TestBase/ObjectStorageTestBase.cs | 18 ++++++++++++------ 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Base/src/DataStructures/ObjectData.cs b/Base/src/DataStructures/ObjectData.cs index 73ed37078..0a6719d77 100644 --- a/Base/src/DataStructures/ObjectData.cs +++ b/Base/src/DataStructures/ObjectData.cs @@ -26,4 +26,9 @@ public record ObjectData /// Id of the result /// public required string ResultId; + + /// + /// Id of the session + /// + public required string SessionId; } diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index 2c57918bd..9d033b550 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -282,7 +282,8 @@ public async Task> CreateResults(string var add = await objectStorage_.AddOrUpdateAsync(new ObjectData { - ResultId = result.ResultId, + ResultId = result.ResultId, + SessionId = rc.request.SessionId, }, new List> { @@ -328,7 +329,8 @@ public async Task> NotifyResultData(string toke var addTask = objectStorage_.AddOrUpdateAsync(new ObjectData { - ResultId = result, + ResultId = result, + SessionId = sessionData_.SessionId, }, channel.Reader.ReadAllAsync(cancellationToken), cancellationToken); diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index 5a815e1bf..a98978888 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -175,7 +175,8 @@ public override async Task CreateResults(CreateResultsReq var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData { - ResultId = resultId, + ResultId = resultId, + SessionId = request.SessionId, }, new List> { @@ -344,7 +345,8 @@ public override async Task UploadResultData(IAsyncStre var (opaqueId, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData { - ResultId = id.ResultId, + ResultId = id.ResultId, + SessionId = id.SessionId, }, requestStream.ReadAllAsync(context.CancellationToken) .Select(r => r.DataChunk.Memory), diff --git a/Common/src/gRPC/Services/Submitter.cs b/Common/src/gRPC/Services/Submitter.cs index 10804c106..fab7bfb2c 100644 --- a/Common/src/gRPC/Services/Submitter.cs +++ b/Common/src/gRPC/Services/Submitter.cs @@ -397,7 +397,8 @@ public async Task SetResult(string sessionId, var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData { - ResultId = key, + ResultId = key, + SessionId = sessionId, }, chunks, cancellationToken) @@ -454,7 +455,8 @@ public async Task> CreateTasks(string taskRequest.DataDependencies.ToList())); payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(new ObjectData { - ResultId = payloadId, + ResultId = payloadId, + SessionId = sessionId, }, taskRequest.PayloadChunks, cancellationToken)); diff --git a/Common/tests/Pollster/AgentTest.cs b/Common/tests/Pollster/AgentTest.cs index 09162b52c..4420526a9 100644 --- a/Common/tests/Pollster/AgentTest.cs +++ b/Common/tests/Pollster/AgentTest.cs @@ -607,7 +607,8 @@ public async Task GetResourceDataShouldSucceed() var (id, size) = await holder.ObjectStorage.AddOrUpdateAsync(new ObjectData { - ResultId = "", + ResultId = "ResultId", + SessionId = "SessionId", }, new List { diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 0fc8bc544..7b06586ff 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -57,7 +57,8 @@ public async Task SetUp() }; datakey1_ = (await ObjectStorage!.AddOrUpdateAsync(new ObjectData { - ResultId = string.Empty, + ResultId = "ResultId", + SessionId = "SessionId", }, dataBytesList.ToAsyncEnumerable()) .ConfigureAwait(false)).id; @@ -68,7 +69,8 @@ public async Task SetUp() }; datakey2_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData { - ResultId = string.Empty, + ResultId = "ResultId", + SessionId = "SessionId", }, dataBytesList.ToAsyncEnumerable()) .ConfigureAwait(false)).id; @@ -79,7 +81,8 @@ public async Task SetUp() }; datakeyEmpty_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData { - ResultId = string.Empty, + ResultId = "ResultId", + SessionId = "SessionId", }, dataBytesList.ToAsyncEnumerable()) .ConfigureAwait(false)).id; @@ -152,7 +155,8 @@ public async Task AddValuesAsyncWithoutChunkShouldWork() { var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData { - ResultId = string.Empty, + ResultId = "ResultId", + SessionId = "SessionId", }, AsyncEnumerable.Empty>()) .ConfigureAwait(false); @@ -191,7 +195,8 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) { var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData { - ResultId = string.Empty, + ResultId = "ResultId", + SessionId = "SessionId", }, inputChunks.ToAsyncEnumerable() .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) @@ -322,7 +327,8 @@ public async Task DeleteDeleteTwiceShouldSucceed() var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData { - ResultId = string.Empty, + ResultId = "ResultId", + SessionId = "SessionId", }, listChunks.ToAsyncEnumerable()) .ConfigureAwait(false); From 6fc4cb7944eca2c7b5a90649d3115ab1b891806a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 5 Dec 2024 10:03:48 +0100 Subject: [PATCH 09/15] test: rework dataprefetcher tests because Moq does not support anonymous types --- Common/tests/Pollster/DataPrefetcherTest.cs | 163 ++++++++++++-------- 1 file changed, 96 insertions(+), 67 deletions(-) diff --git a/Common/tests/Pollster/DataPrefetcherTest.cs b/Common/tests/Pollster/DataPrefetcherTest.cs index 450deafaa..032c5b83c 100644 --- a/Common/tests/Pollster/DataPrefetcherTest.cs +++ b/Common/tests/Pollster/DataPrefetcherTest.cs @@ -33,6 +33,7 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -56,6 +57,88 @@ public virtual void TearDown() private ActivitySource? activitySource_; + private class CustomGetResultTable : IResultTable + { + private readonly List resultIds_; + private readonly string sessionId_; + + + public CustomGetResultTable(string sessionId, + List resultIds) + { + sessionId_ = sessionId; + resultIds_ = resultIds; + } + + public Task Check(HealthCheckTag tag) + => throw new NotImplementedException(); + + public Task Init(CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public ILogger Logger + => NullLogger.Instance; + + public Task ChangeResultOwnership(string oldTaskId, + IEnumerable requests, + CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public Task Create(ICollection results, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task AddTaskDependencies(IDictionary> dependencies, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task DeleteResult(string key, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task DeleteResults(string sessionId, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public IAsyncEnumerable GetResults(Expression> filter, + Expression> convertor, + CancellationToken cancellationToken = default) + => resultIds_.Select(s => convertor.Compile() + .Invoke(new Result(sessionId_, + s, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(s)))) + .ToAsyncEnumerable(); + + public Task<(IEnumerable results, int totalCount)> ListResultsAsync(Expression> filter, + Expression> orderField, + bool ascOrder, + int page, + int pageSize, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task SetTaskOwnership(ICollection<(string resultId, string taskId)> requests, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task UpdateOneResult(string resultId, + UpdateDefinition updates, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task UpdateManyResults(Expression> filter, + UpdateDefinition updates, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + } + [Test] public async Task EmptyPayloadAndOneDependency() { @@ -80,38 +163,16 @@ public async Task EmptyPayloadAndOneDependency() Convert.FromBase64String("3333"), Convert.FromBase64String("4444"), }.ToAsyncEnumerable()); - var mockResultTable = new Mock(); - mockResultTable.Setup(x => x.GetResults(It.IsAny>>(), - It.IsAny>>(), - CancellationToken.None)) - .Returns(new List - { - new(sessionId, - dependency1, - "", - "", - "", - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - 100, - Encoding.UTF8.GetBytes(dependency1)), - new(sessionId, - payloadId, - "", - "", - "", - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - 100, - Encoding.UTF8.GetBytes(payloadId)), - }.ToAsyncEnumerable()); var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, - mockResultTable.Object, + new CustomGetResultTable(sessionId, + new List + { + dependency1, + payloadId, + }), activitySource_, loggerFactory.CreateLogger()); @@ -194,48 +255,16 @@ public async Task EmptyPayloadAndNoDependenciesStateMachine() Convert.FromBase64String("4444"), }.ToAsyncEnumerable()); - var mockResultTable = new Mock(); - mockResultTable.Setup(x => x.GetResults(It.IsAny>>(), - It.IsAny>>(), - CancellationToken.None)) - .Returns(new List - { - new(sessionId, - dependency1, - "", - "", - "", - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - 100, - Encoding.UTF8.GetBytes(dependency1)), - new(sessionId, - dependency2, - "", - "", - "", - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - 100, - Encoding.UTF8.GetBytes(dependency2)), - new(sessionId, - payloadId, - "", - "", - "", - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - 100, - Encoding.UTF8.GetBytes(payloadId)), - }.ToAsyncEnumerable()); - var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, - mockResultTable.Object, + new CustomGetResultTable(sessionId, + new List + { + dependency1, + dependency2, + payloadId, + }), activitySource_, loggerFactory.CreateLogger()); From dc41c71a74e9d9d5921ad6cbcf29ea3a6287c980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 5 Dec 2024 15:48:23 +0100 Subject: [PATCH 10/15] refactor: Object storage IDs are not limited to GUID --- Adaptors/LocalStorage/src/ObjectStorage.cs | 12 +++++++----- Adaptors/Memory/src/ObjectStorage.cs | 12 +++++++----- Adaptors/Redis/src/ObjectStorage.cs | 18 +++++++++++------- Adaptors/S3/src/ObjectStorage.cs | 17 ++++++++++------- Common/tests/TestBase/ObjectStorageTestBase.cs | 9 +++++---- 5 files changed, 40 insertions(+), 28 deletions(-) diff --git a/Adaptors/LocalStorage/src/ObjectStorage.cs b/Adaptors/LocalStorage/src/ObjectStorage.cs index 2f9d99ac6..7ef35013f 100644 --- a/Adaptors/LocalStorage/src/ObjectStorage.cs +++ b/Adaptors/LocalStorage/src/ObjectStorage.cs @@ -19,6 +19,7 @@ using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -97,10 +98,11 @@ public Task Check(HealthCheckTag tag) IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { - var id = Guid.NewGuid(); long size = 0; + var key = Guid.NewGuid() + .ToString(); var filename = Path.Combine(path_, - id.ToString()); + key); // Write to temporary file @@ -134,14 +136,14 @@ public Task Check(HealthCheckTag tag) await file.FlushAsync(cancellationToken) .ConfigureAwait(false); - return (id.ToByteArray(), size); + return (Encoding.UTF8.GetBytes(key), size); } /// public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var key = new Guid(id).ToString(); + var key = Encoding.UTF8.GetString(id); var filename = Path.Combine(path_, key); @@ -191,7 +193,7 @@ public async Task TryDeleteAsync(IEnumerable ids, { foreach (var id in ids) { - var key = new Guid(id).ToString(); + var key = Encoding.UTF8.GetString(id); await TryDeleteAsync(key, cancellationToken) .ConfigureAwait(false); diff --git a/Adaptors/Memory/src/ObjectStorage.cs b/Adaptors/Memory/src/ObjectStorage.cs index 2495ef11f..1e91b7e52 100644 --- a/Adaptors/Memory/src/ObjectStorage.cs +++ b/Adaptors/Memory/src/ObjectStorage.cs @@ -19,6 +19,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -55,7 +56,8 @@ public Task Check(HealthCheckTag tag) { var array = new List(); - var id = Guid.NewGuid(); + var key = Guid.NewGuid() + .ToString(); await foreach (var val in valueChunks.WithCancellation(cancellationToken) .ConfigureAwait(false)) @@ -63,9 +65,9 @@ public Task Check(HealthCheckTag tag) array.AddRange(val.ToArray()); } - store_[id.ToString()] = array.ToArray(); + store_[key] = array.ToArray(); - return (id.ToByteArray(), array.Count); + return (Encoding.UTF8.GetBytes(key), array.Count); } #pragma warning disable CS1998 @@ -73,7 +75,7 @@ public async IAsyncEnumerable GetValuesAsync(byte[] id, #pragma warning restore CS1998 [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var key = new Guid(id).ToString(); + var key = Encoding.UTF8.GetString(id); if (!store_.TryGetValue(key, out var value)) { @@ -91,7 +93,7 @@ public Task TryDeleteAsync(IEnumerable ids, { foreach (var id in ids) { - var key = new Guid(id).ToString(); + var key = Encoding.UTF8.GetString(id); store_.TryRemove(key, out _); diff --git a/Adaptors/Redis/src/ObjectStorage.cs b/Adaptors/Redis/src/ObjectStorage.cs index e80c27f6d..bb45932c1 100644 --- a/Adaptors/Redis/src/ObjectStorage.cs +++ b/Adaptors/Redis/src/ObjectStorage.cs @@ -19,6 +19,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -90,9 +91,10 @@ public Task Check(HealthCheckTag tag) IAsyncEnumerable> valueChunks, CancellationToken cancellationToken = default) { - var id = Guid.NewGuid(); + var key = Guid.NewGuid() + .ToString(); + var storageNameKey = objectStorageName_ + key; long size = 0; - var storageNameKey = objectStorageName_ + id; var idx = 0; var taskList = new List(); @@ -111,14 +113,14 @@ public Task Check(HealthCheckTag tag) await taskList.WhenAll() .ConfigureAwait(false); - return (id.ToByteArray(), size); + return (Encoding.UTF8.GetBytes(key), size); } /// public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var key = new Guid(id); + var key = Encoding.UTF8.GetString(id); var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); @@ -146,13 +148,15 @@ public async IAsyncEnumerable GetValuesAsync(byte[] /// public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) - => await ids.ParallelForEach(key => TryDeleteAsync(new Guid(key).ToString(), - cancellationToken)) + => await ids.ParallelForEach(id => TryDeleteAsync(id, + cancellationToken)) .ConfigureAwait(false); - private async Task TryDeleteAsync(string key, + private async Task TryDeleteAsync(byte[] id, CancellationToken cancellationToken = default) { + var key = Encoding.UTF8.GetString(id); + var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); diff --git a/Adaptors/S3/src/ObjectStorage.cs b/Adaptors/S3/src/ObjectStorage.cs index 939a2bd4c..d63acb17a 100644 --- a/Adaptors/S3/src/ObjectStorage.cs +++ b/Adaptors/S3/src/ObjectStorage.cs @@ -20,6 +20,7 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -98,7 +99,7 @@ public Task Check(HealthCheckTag tag) public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var key = new Guid(id); + var key = Encoding.UTF8.GetString(id); var objectStorageFullName = $"{objectStorageName_}{key}"; try @@ -173,8 +174,9 @@ await s3Client_.GetObjectAsync(options_.BucketName, { 0, }; - var id = Guid.NewGuid(); - var objectStorageFullName = $"{objectStorageName_}{id.ToString()}"; + var key = Guid.NewGuid() + .ToString(); + var objectStorageFullName = $"{objectStorageName_}{key}"; logger_.LogDebug("Upload object"); var initRequest = new InitiateMultipartUploadRequest @@ -226,19 +228,20 @@ await s3Client_.AbortMultipartUploadAsync(abortMpuRequest, throw; } - return (id.ToByteArray(), sizeBox[0]); + return (Encoding.UTF8.GetBytes(key), sizeBox[0]); } /// public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) - => await ids.ParallelForEach(key => TryDeleteAsync(new Guid(key).ToString(), - cancellationToken)) + => await ids.ParallelForEach(id => TryDeleteAsync(id, + cancellationToken)) .ConfigureAwait(false); - private async Task TryDeleteAsync(string key, + private async Task TryDeleteAsync(byte[] id, CancellationToken cancellationToken = default) { + var key = Encoding.UTF8.GetString(id); var objectStorageFullName = $"{objectStorageName_}{key}"; var objectDeleteRequest = new DeleteObjectRequest diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 7b06586ff..ef1b37cd3 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -224,8 +224,7 @@ public async Task GetValuesAsyncShouldFail() { if (RunTests) { - var id = Guid.NewGuid() - .ToByteArray(); + var id = Encoding.UTF8.GetBytes("IdThatShouldFail"); var data = new List(); try @@ -266,7 +265,8 @@ public async Task PayloadShouldBeEqual() var str = Encoding.ASCII.GetString(data.ToArray()); Console.WriteLine(str); - Assert.IsTrue(str.SequenceEqual("AAAABBBB")); + Assert.AreEqual("AAAABBBB", + str); } } @@ -286,7 +286,8 @@ public async Task Payload2ShouldBeEqual() var str = Encoding.ASCII.GetString(data.ToArray()); Console.WriteLine(str); - Assert.IsTrue(str.SequenceEqual("AAAABBBBCCCCDDDD")); + Assert.AreEqual("AAAABBBBCCCCDDDD", + str); } } From b1d31a9d99ff6c2000312fdb4bd0a3eee1e7f03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 12 Dec 2024 11:07:08 +0100 Subject: [PATCH 11/15] docs: add database migration procedure --- .../0.installation/5.version-migration.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .docs/content/0.installation/5.version-migration.md diff --git a/.docs/content/0.installation/5.version-migration.md b/.docs/content/0.installation/5.version-migration.md new file mode 100644 index 000000000..c8b25674a --- /dev/null +++ b/.docs/content/0.installation/5.version-migration.md @@ -0,0 +1,51 @@ +# How to migrate ArmoniK.Core dependencies during upgrade ? + +## 0.28.x -> 0.29.x + +### Database + +This version changes the structure of a Result in the database. It introduces a new field called `OpaqueId` which holds the identifier of its associated value in the Object Storage. Previously, the ResultId was used. The following MongoDB request converts the ResultId into the OpaqueId to support the new implementation. + +```js +db.Result.updateMany({}, + [{ + $addFields: { + OpaqueId: { + $function: { + body: function(data) { + const base64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + const bytes = []; + for (let i = 0; i < data.length; i++) { + bytes.push(data.charCodeAt(i)); + } + + let base64 = ''; + let i = 0; + while (i < bytes.length) { + let byte1 = bytes[i++] || 0; + let byte2 = bytes[i++] || 0; + let byte3 = bytes[i++] || 0; + + let enc1 = byte1 >> 2; + let enc2 = ((byte1 & 3) << 4) | (byte2 >> 4); + let enc3 = ((byte2 & 15) << 2) | (byte3 >> 6); + let enc4 = byte3 & 63; + + if (isNaN(byte2)) { + enc3 = enc4 = 64; + } else if (isNaN(byte3)) { + enc4 = 64; + } + + base64 += base64Chars[enc1] + base64Chars[enc2] + base64Chars[enc3] + base64Chars[enc4]; + } + + return BinData(0, base64); + }, + args: ["$_id"], + lang: "js" + } + } + } + }]) +``` From f781f895f85b852fca3daa04c80ef5ed8ddbef5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 12 Dec 2024 17:52:38 +0100 Subject: [PATCH 12/15] fix: remove Redis package from Embed plugin --- Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj index da557fc65..05f2744e5 100644 --- a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj @@ -29,7 +29,6 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - From 9bdb521deaca3f296c42cd1d4c22736cd6573b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 12 Dec 2024 17:54:36 +0100 Subject: [PATCH 13/15] docs: update wrong summaries for new Object Storages --- Adaptors/Embed/src/ObjectBuilder.cs | 2 +- Adaptors/LocalStorage/src/ObjectBuilder.cs | 2 +- Adaptors/Redis/src/ObjectBuilder.cs | 2 +- Adaptors/S3/src/ObjectBuilder.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Adaptors/Embed/src/ObjectBuilder.cs b/Adaptors/Embed/src/ObjectBuilder.cs index cd41e8fc0..28f7063ef 100644 --- a/Adaptors/Embed/src/ObjectBuilder.cs +++ b/Adaptors/Embed/src/ObjectBuilder.cs @@ -27,7 +27,7 @@ namespace ArmoniK.Core.Adapters.Embed; /// -/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// Class for building Embed instance and Object interfaces through Dependency Injection /// [PublicAPI] public class ObjectBuilder : IDependencyInjectionBuildable diff --git a/Adaptors/LocalStorage/src/ObjectBuilder.cs b/Adaptors/LocalStorage/src/ObjectBuilder.cs index b29e4563b..7de022ad8 100644 --- a/Adaptors/LocalStorage/src/ObjectBuilder.cs +++ b/Adaptors/LocalStorage/src/ObjectBuilder.cs @@ -27,7 +27,7 @@ namespace ArmoniK.Core.Adapters.LocalStorage; /// -/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// Class for building Local Storage instance and Object interfaces through Dependency Injection /// [PublicAPI] public class ObjectBuilder : IDependencyInjectionBuildable diff --git a/Adaptors/Redis/src/ObjectBuilder.cs b/Adaptors/Redis/src/ObjectBuilder.cs index af19fec07..0374b48c9 100644 --- a/Adaptors/Redis/src/ObjectBuilder.cs +++ b/Adaptors/Redis/src/ObjectBuilder.cs @@ -33,7 +33,7 @@ namespace ArmoniK.Core.Adapters.Redis; /// -/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// Class for building Redis instance and Object interfaces through Dependency Injection /// [PublicAPI] public class ObjectBuilder : IDependencyInjectionBuildable diff --git a/Adaptors/S3/src/ObjectBuilder.cs b/Adaptors/S3/src/ObjectBuilder.cs index 17df4d945..599a64846 100644 --- a/Adaptors/S3/src/ObjectBuilder.cs +++ b/Adaptors/S3/src/ObjectBuilder.cs @@ -29,7 +29,7 @@ namespace ArmoniK.Core.Adapters.S3; /// -/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection +/// Class for building S3 instance and Object interfaces through Dependency Injection /// [PublicAPI] public class ObjectBuilder : IDependencyInjectionBuildable From 5b1ddc573f67b9af041cd23d24de84ec262be8b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Thu, 12 Dec 2024 17:56:19 +0100 Subject: [PATCH 14/15] refactor: use ViewSelect when suitable --- Common/src/Pollster/Agent.cs | 8 ++++---- Common/src/gRPC/Services/GrpcResultsService.cs | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index 9d033b550..780b840c1 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -297,8 +297,9 @@ public async Task> CreateResults(string .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results.Select(tuple => tuple.result) - .AsICollection(), + var ids = results.ViewSelect(tuple => tuple.result); + + await resultTable_.Create(ids, cancellationToken) .ConfigureAwait(false); @@ -308,8 +309,7 @@ await resultTable_.Create(results.Select(tuple => tuple.result) result.add); } - return results.Select(tuple => tuple.result) - .AsICollection(); + return ids; } /// diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index a98978888..8d99b6ccc 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -199,8 +199,7 @@ public override async Task CreateResults(CreateResultsReq .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results.Select(tuple => tuple.Item1) - .AsICollection(), + await resultTable_.Create(results.ViewSelect(tuple => tuple.Item1), context.CancellationToken) .ConfigureAwait(false); From d6e75c71b58e0f5bca9405f289a47f53f31761c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gurhem?= Date: Mon, 16 Dec 2024 08:50:01 +0100 Subject: [PATCH 15/15] refactor: Embed object storage improvements --- Adaptors/Embed/src/ObjectStorage.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Adaptors/Embed/src/ObjectStorage.cs b/Adaptors/Embed/src/ObjectStorage.cs index bc4ea87b5..8bb28d693 100644 --- a/Adaptors/Embed/src/ObjectStorage.cs +++ b/Adaptors/Embed/src/ObjectStorage.cs @@ -17,7 +17,7 @@ using System; using System.Collections.Generic; -using System.Runtime.CompilerServices; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -67,18 +67,17 @@ public Task Check(HealthCheckTag tag) await foreach (var val in valueChunks.WithCancellation(cancellationToken) .ConfigureAwait(false)) { - array.AddRange(val.ToArray()); + array.AddRange(val.Span); } return (array.ToArray(), array.Count); } /// - public async IAsyncEnumerable GetValuesAsync(byte[] id, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - yield return id; - } + public IAsyncEnumerable GetValuesAsync(byte[] id, + CancellationToken cancellationToken = default) + => AsyncEnumerable.Repeat(id, + 1); /// public Task TryDeleteAsync(IEnumerable ids,