diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36a1fff9..e6b5dba3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: - name: Install ReSharper run: | - dotnet tool install -g JetBrains.ReSharper.GlobalTools --version 2022.2.2 + dotnet tool install -g JetBrains.ReSharper.GlobalTools --version 2023.2.0 - name: Restore run: | @@ -63,6 +63,7 @@ jobs: jb cleanupcode --exclude="**.props" ArmoniK.Extensions.Csharp.sln - name: Check Diff + id: check-diff run: | DIFF="$(git diff --name-only)" @@ -76,6 +77,17 @@ jobs: exit 1 fi + - name: Generate patch + if: ${{ failure() && steps.check-diff.conclusion == 'failure' }} + run: | + git diff > patch-csharp.diff + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3 + if: ${{ failure() && steps.check-diff.conclusion == 'failure' }} + with: + name: patch-csharp + path: ./patch-csharp.diff + buildProjects: runs-on: ubuntu-latest needs: @@ -218,7 +230,7 @@ jobs: tls: ${{ matrix.tls }} mtls: ${{ matrix.mtls }} ext-csharp-version: ${{ needs.versionning.outputs.version }} - core-version: 0.14.3 + core-version: 0.15.0 - name: Setup hosts file run : echo -e "$(kubectl get svc ingress -n armonik -o jsonpath={.status.loadBalancer.ingress[0].ip})\tarmonik.local" | sudo tee -a /etc/hosts @@ -266,6 +278,87 @@ jobs: name: "IntegrationTests tls:${{ matrix.tls }} mtls:${{ matrix.mtls }} val:${{ matrix.sslvalidation }} ca:${{ matrix.useca }}" path: ./Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/TestResults/test-results.trx reporter: dotnet-trx + + - name: Show logs + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + AWS_DEFAULT_OUTPUT: json + run: | + set -e + cd infra + mkdir -p end2end/infra/generated \ + end2end/infra/tfstates \ + end2end/app + if [ -e infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json ] ; then + cp infrastructure/quick-deploy/localhost/all-in-one/generated/armonik-output.json end2end/infra/generated + fi + cp infrastructure/quick-deploy/localhost/all-in-one/generated/terraform.tfstate end2end/infra/tfstates + sudo cp -rL /var/log/pods/armonik_* end2end/app + sudo chown $USER -R end2end + tar -czf end2end-${{ matrix.tls }}-${{ matrix.mtls }}-${{ matrix.sslvalidation }}-${{ matrix.useca }}.tar.gz end2end + aws s3 cp end2end-${{ matrix.tls }}-${{ matrix.mtls }}-${{ matrix.sslvalidation }}-${{ matrix.useca }}.tar.gz s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/extcsharp-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/end2end-${{ matrix.tls }}-${{ matrix.mtls }}-${{ matrix.sslvalidation }}-${{ matrix.useca }}.tar.gz + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v3 + with: + name: end2end-${{ matrix.tls }}-${{ matrix.mtls }}-${{ matrix.sslvalidation }}-${{ matrix.useca }}.tar.gz + path: end2end-${{ matrix.tls }}-${{ matrix.mtls }}-${{ matrix.sslvalidation }}-${{ matrix.useca }}.tar.gz + retention-days: 2 + + + test-container: + runs-on: ubuntu-latest + needs: + - versionning + - buildWorkerEnd2End + env: + VERSION: ${{ needs.versionning.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + + - name: Install Dependencies + uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main + with: + docker: true + terraform: true + k3s: true + aws: true + + - name: Checkout Infra + uses: actions/checkout@v3 + with: + repository: aneoconsulting/ArmoniK + path: infra + + - name: Build Docker image + run: | + docker build -t test_image:latest --build-arg WORKER_DLL_IMAGE=dockerhubaneo/armonik_worker_dll:$VERSION -f Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Dockerfile . + + - name: Change version + run: | + cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.extcsharp=$version | .armonik_images.extcsharp=["test_image"]' > .versions.tfvars.json + mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json + + - id: deploy + name: Deploy + uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main + with: + working-directory: ${{ github.workspace }}/infra + type: localhost + ext-csharp-version: ${{ needs.versionning.outputs.version }} + core-version: 0.15.0 + + - name: Run Test + run: | + cd Tests + bash -x ./endToEndTests.sh canMerge: needs: diff --git a/.github/workflows/build_documentations.yml b/.github/workflows/build_documentations.yml new file mode 100644 index 00000000..e20499e4 --- /dev/null +++ b/.github/workflows/build_documentations.yml @@ -0,0 +1,34 @@ +name: Build Documentation + +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Get dotnet sdk + run: | + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + rm packages-microsoft-prod.deb + sudo apt-get update + sudo apt-get install -y dotnet-sdk-7.0 + - name: Get docfx + run: | + dotnet tool update -g docfx + - name: Build docs + run: | + docfx Documentation/docfx.json + + - name: Deploy docfx documentation + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: Documentation/_site diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml deleted file mode 100644 index 298bc07b..00000000 --- a/.github/workflows/deploy-docs.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Deploy Docs to Pages - -on: - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - generate-docs: - name: Generate Docs - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - run: npm i -g pnpm @antfu/ni - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: "18" - cache: "pnpm" - cache-dependency-path: ".docs/pnpm-lock.yaml" - - - name: Install dependencies - run: cd .docs && nci - - - name: Use NODE_ENV=production - run: echo "NODE_ENV=production" >> $GITHUB_ENV - - - name: Set the site URL - run: echo "NUXT_PUBLIC_SITE_URL=https://aneoconsulting.github.io/ArmoniK.Extensions.Csharp" >> $GITHUB_ENV - - - name: Static HTML export with Nuxt - run: cd .docs && nr generate - - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - with: - path: .docs/.output/public - - deploy: - needs: generate-docs - name: Deploy to GitHub Pages - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@af48cf94a42f2c634308b1c9dc0151830b6f190a # v2 diff --git a/.github/workflows/validate-docs-generation.yml b/.github/workflows/validate-docs-generation.yml deleted file mode 100644 index b2a59448..00000000 --- a/.github/workflows/validate-docs-generation.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Validate Docs Generation - -on: - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - generate-docs: - name: Generate Docs - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - run: npm i -g pnpm @antfu/ni - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: "pnpm" - cache-dependency-path: ".docs/pnpm-lock.yaml" - - - name: Install dependencies - run: cd .docs && nci - - - name: Use NODE_ENV=production - run: echo "NODE_ENV=production" >> $GITHUB_ENV - - - name: Static HTML export with Nuxt - run: cd .docs && nr generate - - - lint-markdown: - name: Lint Markdown - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - run: npm i -g pnpm @antfu/ni - - - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: pnpm - cache-dependency-path: ".docs/pnpm-lock.yaml" - - - run: cd .docs && nci - - - run: cd .docs && nr lint:md - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - run: npm i -g pnpm @antfu/ni - - - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: pnpm - cache-dependency-path: ".docs/pnpm-lock.yaml" - - - run: cd .docs && nci - - - run: cd .docs && nr lint diff --git a/ArmoniK.Extensions.Csharp.sln b/ArmoniK.Extensions.Csharp.sln index f20bf381..ff55adbb 100644 --- a/ArmoniK.Extensions.Csharp.sln +++ b/ArmoniK.Extensions.Csharp.sln @@ -49,16 +49,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Symphony", "Symphony", "{E5 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DLLWorker", "DLLWorker", "{AB285F22-A32F-4C5C-A6B3-294E347BFFAE}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Symphony", "Symphony", "{2343D895-0821-4EBB-A56D-C58F817D5FF4}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Unified", "Unified", "{34DA3A29-FD3C-462B-BD35-38D699C4D901}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.EndToEndTests.Common", "Tests\ArmoniK.EndToEndTests\ArmoniK.EndToEndTests.Common\ArmoniK.EndToEndTests.Common.csproj", "{E7AE7482-42A7-4113-AB1E-EBECE53AF6CA}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.EndToEndTests.Client", "Tests\ArmoniK.EndToEndTests\ArmoniK.EndToEndTests.Client\ArmoniK.EndToEndTests.Client.csproj", "{7E5AE5BF-099E-4E00-B7CB-1C80FDC7C193}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.EndToEndTests.Worker", "Tests\ArmoniK.EndToEndTests\ArmoniK.EndToEndTests.Worker\ArmoniK.EndToEndTests.Worker.csproj", "{B960962F-4CB1-480D-8D10-9DE2990896B7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.DevelopmentKit.Common.Tests", "Tests\ArmoniK.DevelopmentKit.Common.Tests\ArmoniK.DevelopmentKit.Common.Tests.csproj", "{84BB2691-33F0-45B4-8D63-0ECF82708CFC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -109,6 +107,10 @@ Global {B960962F-4CB1-480D-8D10-9DE2990896B7}.Debug|Any CPU.Build.0 = Debug|Any CPU {B960962F-4CB1-480D-8D10-9DE2990896B7}.Release|Any CPU.ActiveCfg = Release|Any CPU {B960962F-4CB1-480D-8D10-9DE2990896B7}.Release|Any CPU.Build.0 = Release|Any CPU + {84BB2691-33F0-45B4-8D63-0ECF82708CFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84BB2691-33F0-45B4-8D63-0ECF82708CFC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84BB2691-33F0-45B4-8D63-0ECF82708CFC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84BB2691-33F0-45B4-8D63-0ECF82708CFC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -132,11 +134,10 @@ Global {E6F98032-63BB-41CC-8E71-5E5DB54C3A87} = {9EFC0507-3071-4F7D-BFDD-6D99880D80C4} {E5766860-034C-4F83-907B-922205DD2E0E} = {9EFC0507-3071-4F7D-BFDD-6D99880D80C4} {AB285F22-A32F-4C5C-A6B3-294E347BFFAE} = {9EFC0507-3071-4F7D-BFDD-6D99880D80C4} - {2343D895-0821-4EBB-A56D-C58F817D5FF4} = {29622951-3654-41F0-9393-11D6737FD1F0} - {34DA3A29-FD3C-462B-BD35-38D699C4D901} = {29622951-3654-41F0-9393-11D6737FD1F0} {E7AE7482-42A7-4113-AB1E-EBECE53AF6CA} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} {7E5AE5BF-099E-4E00-B7CB-1C80FDC7C193} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} {B960962F-4CB1-480D-8D10-9DE2990896B7} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} + {84BB2691-33F0-45B4-8D63-0ECF82708CFC} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1285A466-2AF6-43E6-8DCC-2F93A5D5F02E} diff --git a/ArmoniK.Extensions.Csharp.sln.DotSettings b/ArmoniK.Extensions.Csharp.sln.DotSettings index 66756f20..14d4fcc0 100644 --- a/ArmoniK.Extensions.Csharp.sln.DotSettings +++ b/ArmoniK.Extensions.Csharp.sln.DotSettings @@ -16,10 +16,12 @@ True True True + True True True True True + True True True True diff --git a/Client/src/Common/ArmoniK.DevelopmentKit.Client.Common.csproj b/Client/src/Common/ArmoniK.DevelopmentKit.Client.Common.csproj index 942955b9..ca1477d2 100644 --- a/Client/src/Common/ArmoniK.DevelopmentKit.Client.Common.csproj +++ b/Client/src/Common/ArmoniK.DevelopmentKit.Client.Common.csproj @@ -1,7 +1,7 @@ - net472;net48;net5.0;net6.0 + netstandard2.0 Library True true @@ -9,9 +9,6 @@ - - - diff --git a/Client/src/Common/Exceptions/ServiceInvocationException.cs b/Client/src/Common/Exceptions/ServiceInvocationException.cs index c4a6447c..aadb5185 100644 --- a/Client/src/Common/Exceptions/ServiceInvocationException.cs +++ b/Client/src/Common/Exceptions/ServiceInvocationException.cs @@ -16,14 +16,18 @@ using System; +using ArmoniK.DevelopmentKit.Client.Common.Status; using ArmoniK.DevelopmentKit.Common; +using JetBrains.Annotations; + namespace ArmoniK.DevelopmentKit.Client.Common.Exceptions; /// /// The service invocation exception. This class wil contain all error information of task or result /// [MarkDownDoc] +[PublicAPI] public class ServiceInvocationException : Exception { private readonly string message_ = "ServiceInvocationException during call function"; diff --git a/Client/src/Common/IServiceInvocationHandler.cs b/Client/src/Common/IServiceInvocationHandler.cs index 901cd93b..ef4fca26 100644 --- a/Client/src/Common/IServiceInvocationHandler.cs +++ b/Client/src/Common/IServiceInvocationHandler.cs @@ -17,6 +17,8 @@ using ArmoniK.DevelopmentKit.Client.Common.Exceptions; using ArmoniK.DevelopmentKit.Common; +using JetBrains.Annotations; + namespace ArmoniK.DevelopmentKit.Client.Common; /// @@ -24,6 +26,7 @@ namespace ArmoniK.DevelopmentKit.Client.Common; /// in the method LocalExecute, Execute or Submit /// [MarkDownDoc] +[PublicAPI] public interface IServiceInvocationHandler { /// @@ -37,8 +40,8 @@ void HandleError(ServiceInvocationException e, /// /// The callBack method which has to be implemented to retrieve response from the server /// - /// The object receive from the server as result the method called by the client - /// The task identifier which has invoke the response callBack - void HandleResponse(object response, - string taskId); + /// The object received from the server as the result of the method called by the client + /// The task identifier triggering the callBack + void HandleResponse(object? response, + string taskId); } diff --git a/Client/src/Common/Portability/IsExternalInit.cs b/Client/src/Common/Portability/IsExternalInit.cs new file mode 100644 index 00000000..a0c89d77 --- /dev/null +++ b/Client/src/Common/Portability/IsExternalInit.cs @@ -0,0 +1,25 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2023. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#if NETFRAMEWORK || NETSTANDARD +// This type is required to use initializers when compiling to framework +namespace System.Runtime.CompilerServices; + +internal static class IsExternalInit +{ +} +#endif diff --git a/Client/src/Common/Properties.cs b/Client/src/Common/Properties.cs index 5c95c9c5..5c8604e9 100644 --- a/Client/src/Common/Properties.cs +++ b/Client/src/Common/Properties.cs @@ -21,6 +21,8 @@ using Google.Protobuf.WellKnownTypes; +using JetBrains.Annotations; + using Microsoft.Extensions.Configuration; namespace ArmoniK.DevelopmentKit.Client.Common; @@ -32,6 +34,8 @@ namespace ArmoniK.DevelopmentKit.Client.Common; /// The ssl mTLS certificate if needed to connect to the control plane /// [MarkDownDoc] +// TODO: check all setter and mark the required as PublicApi +// TODO: to be reworked to allow all options from API and add other elements. public class Properties { /// @@ -54,15 +58,17 @@ public class Properties /// /// The default configuration to submit task in a Session /// - public static TaskOptions DefaultTaskOptions = new() - { - MaxDuration = new Duration - { - Seconds = 300000, - }, - MaxRetries = 3, - Priority = 1, - }; + // TODO: define [PublicApi] ? + // ReSharper disable once UnusedMember.Global + public static readonly TaskOptions DefaultTaskOptions = new() + { + MaxDuration = new Duration + { + Seconds = 300000, + }, + MaxRetries = 3, + Priority = 1, + }; /// @@ -77,14 +83,16 @@ public class Properties /// The client certificate in a P12/Pkcs12/PFX format /// The Server certificate file to validate mTLS /// Disable the ssl strong validation of ssl certificate (default : enable => true) + // TODO: define [PublicApi] ? + // ReSharper disable once UnusedMember.Global public Properties(TaskOptions options, - string connectionAddress, + string? connectionAddress, int connectionPort = 0, - string protocol = null, - string clientCertPem = null, - string clientKeyPem = null, - string clientP12 = null, - string caCertPem = null, + string? protocol = null, + string? clientCertPem = null, + string? clientKeyPem = null, + string? clientP12 = null, + string? caCertPem = null, bool? sslValidation = null) : this(new ConfigurationBuilder().AddEnvironmentVariables() .Build(), @@ -119,14 +127,14 @@ public Properties(TaskOptions options, /// public Properties(IConfiguration configuration, TaskOptions options, - string connectionAddress = null, - int connectionPort = 0, - string protocol = null, - string clientCertFilePem = null, - string clientKeyFilePem = null, - string clientP12 = null, - string caCertPem = null, - bool? sslValidation = null, + string? connectionAddress = null, + int connectionPort = 0, + string? protocol = null, + string? clientCertFilePem = null, + string? clientKeyFilePem = null, + string? clientP12 = null, + string? caCertPem = null, + bool? sslValidation = null, TimeSpan retryInitialBackoff = new(), double retryBackoffMultiplier = 0, TimeSpan retryMaxBackoff = new()) @@ -134,10 +142,7 @@ public Properties(IConfiguration configuration, TaskOptions = options; Configuration = configuration; - var sectionGrpc = configuration.GetSection(SectionGrpc) - .Exists() - ? configuration.GetSection(SectionGrpc) - : null; + var sectionGrpc = configuration.GetSection(SectionGrpc); if (connectionAddress != null) { @@ -151,17 +156,36 @@ public Properties(IConfiguration configuration, } else { - ConnectionString = sectionGrpc?[SectionEndPoint]; + ConnectionAddress = string.Empty; // to remove a compiler message for netstandard2.0 + try + { + var connectionString = sectionGrpc.GetValue(SectionEndPoint); + if (!string.IsNullOrEmpty(connectionString)) + { + var uri = new Uri(connectionString); + + Protocol = uri.Scheme; + + ConnectionAddress = uri.Host; + ConnectionPort = uri.Port; + } + } + catch (FormatException e) + { + Console.WriteLine(e); + ConnectionAddress = string.Empty; + ConnectionPort = 0; + } } Protocol = protocol ?? Protocol; - ConfSSLValidation = sslValidation ?? sectionGrpc?[SectionSSlValidation] != "disable"; - TargetNameOverride = sectionGrpc?[SectionTargetNameOverride]; - CaCertFilePem = caCertPem ?? sectionGrpc?[SectionCaCert]; - ClientCertFilePem = clientCertFilePem ?? sectionGrpc?[SectionClientCert]; - ClientKeyFilePem = clientKeyFilePem ?? sectionGrpc?[SectionClientKey]; - ClientP12File = clientP12 ?? sectionGrpc?[SectionClientCertP12]; + ConfSslValidation = sslValidation ?? sectionGrpc[SectionSSlValidation] != "disable"; + TargetNameOverride = sectionGrpc[SectionTargetNameOverride] ?? string.Empty; + CaCertFilePem = caCertPem ?? sectionGrpc[SectionCaCert] ?? string.Empty; + ClientCertFilePem = clientCertFilePem ?? sectionGrpc[SectionClientCert] ?? string.Empty; + ClientKeyFilePem = clientKeyFilePem ?? sectionGrpc[SectionClientKey] ?? string.Empty; + ClientP12File = clientP12 ?? sectionGrpc[SectionClientCertP12] ?? string.Empty; if (retryInitialBackoff != TimeSpan.Zero) { @@ -197,45 +221,56 @@ public Properties(IConfiguration configuration, ConnectionPort = connectionPort; } - //Check if Uri is correct if (string.IsNullOrEmpty(Protocol) || string.IsNullOrEmpty(ConnectionAddress) || ConnectionPort == 0) { throw new ArgumentException($"Issue with the connection point : {ConnectionString}"); } - ControlPlaneUri = new Uri(ConnectionString!); + ControlPlaneUri = new Uri(ConnectionString); } /// /// Set the number of task by buffer /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public int MaxTasksPerBuffer { get; set; } = 500; /// /// Set the number of buffers that can be filled in asynchronous submitAsync /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public int MaxConcurrentBuffers { get; set; } = 1; /// /// TimeSpan to trigger a batch to send the batch of submit /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public TimeSpan? TimeTriggerBuffer { get; set; } = TimeSpan.FromSeconds(10); /// /// The number of channels used for Buffered Submit (Default 1) /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public int MaxParallelChannels { get; set; } = 1; /// /// The control plane url to connect /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public Uri ControlPlaneUri { get; set; } /// /// The path to the CA Root file name /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public string CaCertFilePem { get; set; } /// @@ -256,66 +291,68 @@ public Properties(IConfiguration configuration, /// /// The SSL validation property to disable SSL strong verification /// - public bool ConfSSLValidation { get; } + [PublicAPI] + [Obsolete("Use ConfSslValidation instead")] + // ReSharper disable once InconsistentNaming + public bool ConfSSLValidation + => ConfSslValidation; + + /// + /// The SSL validation property to disable SSL strong verification + /// + public bool ConfSslValidation { get; } /// /// The configuration property to give to the ClientService connector /// + // TODO: mark as [PublicApi] ? + // ReSharper disable once UnusedAutoPropertyAccessor.Global public IConfiguration Configuration { get; } /// /// The connection string building the value Port Protocol and address /// + // TODO: mark as [PublicApi] ? + // ReSharper disable once MemberCanBePrivate.Global public string ConnectionString - { - get => $"{Protocol}://{ConnectionAddress}:{ConnectionPort}"; - set - { - try - { - if (string.IsNullOrEmpty(value)) - { - return; - } - - var uri = new Uri(value); - - Protocol = uri.Scheme; - - ConnectionAddress = uri.Host; - ConnectionPort = uri.Port; - } - catch (FormatException e) - { - Console.WriteLine(e); - throw; - } - } - } + => $"{Protocol}://{ConnectionAddress}:{ConnectionPort}"; /// /// Secure or insecure protocol communication https or http (Default http) /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once MemberCanBePrivate.Global public string Protocol { get; set; } = "http"; /// /// The connection address property to connect to the control plane /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once MemberCanBePrivate.Global public string ConnectionAddress { get; set; } /// /// The option connection port to connect to control plane (Default : 5001) /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once MemberCanBePrivate.Global + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public int ConnectionPort { get; set; } = 5001; /// /// The TaskOptions to pass to the session or the submission session /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once MemberCanBePrivate.Global + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public TaskOptions TaskOptions { get; set; } /// /// The target name of the endpoint when ssl validation is disabled. Automatic if not set. /// + // TODO: mark as [PublicApi] for setter ? + // ReSharper disable once MemberCanBePrivate.Global + // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Global public string TargetNameOverride { get; set; } = ""; /// diff --git a/Client/src/Common/Exceptions/ArmonikStatusCode.cs b/Client/src/Common/Status/ArmonikStatusCode.cs similarity index 90% rename from Client/src/Common/Exceptions/ArmonikStatusCode.cs rename to Client/src/Common/Status/ArmonikStatusCode.cs index 32b60469..2b45be1b 100644 --- a/Client/src/Common/Exceptions/ArmonikStatusCode.cs +++ b/Client/src/Common/Status/ArmonikStatusCode.cs @@ -14,16 +14,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -namespace ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using System; + +using JetBrains.Annotations; + +namespace ArmoniK.DevelopmentKit.Client.Common.Status; /// /// List of status for task and result in Armonik /// +[PublicAPI] public enum ArmonikStatusCode { + /// + /// Unknown status of task or result + /// + Unknown, + /// /// The task is completed but result could not be ready /// + [Obsolete("unused")] TaskCompleted, /// @@ -52,12 +63,8 @@ public enum ArmonikStatusCode ResultNotReady, /// - /// The result is in error and the task could finished without no result + /// The result is in error and the task could finished without result /// + [Obsolete("unused")] ResultError, - - /// - /// Unknown status of task or result - /// - Unknown, } diff --git a/Client/src/Common/Status/ResultStatusCollection.cs b/Client/src/Common/Status/ResultStatusCollection.cs index 11f22923..5fb6ff70 100644 --- a/Client/src/Common/Status/ResultStatusCollection.cs +++ b/Client/src/Common/Status/ResultStatusCollection.cs @@ -16,35 +16,24 @@ using System.Collections.Generic; +using JetBrains.Annotations; + namespace ArmoniK.DevelopmentKit.Client.Common.Status; /// /// List of result status that will be collected during the request GetResultStatus /// -public class ResultStatusCollection -{ - /// - /// List of completed task where the result is ready to be retrieved - /// - public IEnumerable IdsReady { get; set; } = default; - - /// - /// List of task or task result in error - /// - public IEnumerable IdsResultError { get; set; } = default; - - /// - /// List of Unknown TaskIds. There is a heavy error somewhere else in the execution when this list has element - /// - public IEnumerable IdsError { get; set; } = default; - - /// - /// List of result not yet written in database - /// - public IEnumerable IdsNotReady { get; set; } - - /// - /// The list of canceled task - /// - public IEnumerable Canceled { get; set; } -} +/// List of completed task where the result is ready to be retrieved +/// List of task or task result in error +/// +/// List of Unknown TaskIds. There is a heavy error somewhere else in the execution when this list +/// has element +/// +/// List of result not yet written in database +/// List of canceled task +[PublicAPI] +public sealed record ResultStatusCollection(IReadOnlyList IdsReady, + IReadOnlyList IdsResultError, + IReadOnlyList IdsError, + IReadOnlyList IdsNotReady, + IReadOnlyList Canceled); diff --git a/Client/src/Common/Status/ResultStatusData.cs b/Client/src/Common/Status/ResultStatusData.cs index 87aa4a46..9bb80f6b 100644 --- a/Client/src/Common/Status/ResultStatusData.cs +++ b/Client/src/Common/Status/ResultStatusData.cs @@ -16,40 +16,17 @@ using ArmoniK.Api.gRPC.V1; +using JetBrains.Annotations; + namespace ArmoniK.DevelopmentKit.Client.Common.Status; /// -/// Class for storing relation between result id, task id and result status +/// Stores the relation between result id, task id and result status /// -public class ResultStatusData -{ - /// - /// Constructor for the class - /// - /// The id of the result - /// The id of the task producing the result - /// The status of the result - public ResultStatusData(string resultId, - string taskId, - ResultStatus status) - { - ResultId = resultId; - TaskId = taskId; - Status = status; - } - - /// - /// The id of the result - /// - public string ResultId { get; } - - /// - /// The id of the task producing the result - /// - public string TaskId { get; } - - /// - /// The status of the result - /// - public ResultStatus Status { get; } -} +/// The id of the result +/// The id of the task producing the result +/// The status of the result +[PublicAPI] +public sealed record ResultStatusData(string ResultId, + string TaskId, + ResultStatus Status); diff --git a/Client/src/Common/Status/TaskStatusExt.cs b/Client/src/Common/Status/TaskStatusExt.cs new file mode 100644 index 00000000..0a74506e --- /dev/null +++ b/Client/src/Common/Status/TaskStatusExt.cs @@ -0,0 +1,47 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2023. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using ArmoniK.Api.gRPC.V1; + +namespace ArmoniK.DevelopmentKit.Client.Common.Status; + +/// +/// Extends the ArmoniK API TaskStatus type to provide some means for conversion. +/// +public static class TaskStatusExt +{ + /// + /// Converts the status from native API representation to SDK representation + /// + /// the native API status to convert + /// the SDK status + public static ArmonikStatusCode ToArmonikStatusCode(this TaskStatus taskStatus) + => taskStatus switch + { + TaskStatus.Submitted => ArmonikStatusCode.ResultNotReady, + TaskStatus.Timeout => ArmonikStatusCode.TaskTimeout, + TaskStatus.Cancelled => ArmonikStatusCode.TaskCancelled, + TaskStatus.Cancelling => ArmonikStatusCode.TaskCancelled, + TaskStatus.Error => ArmonikStatusCode.TaskFailed, + TaskStatus.Processing => ArmonikStatusCode.ResultNotReady, + TaskStatus.Dispatched => ArmonikStatusCode.ResultNotReady, + TaskStatus.Completed => ArmonikStatusCode.ResultNotReady, + TaskStatus.Creating => ArmonikStatusCode.ResultNotReady, + TaskStatus.Unspecified => ArmonikStatusCode.TaskFailed, + TaskStatus.Processed => ArmonikStatusCode.ResultReady, + _ => ArmonikStatusCode.Unknown, + }; +} diff --git a/Client/src/Common/Submitter/BaseClientSubmitter.cs b/Client/src/Common/Submitter/BaseClientSubmitter.cs index e339f982..f5bfa5b3 100644 --- a/Client/src/Common/Submitter/BaseClientSubmitter.cs +++ b/Client/src/Common/Submitter/BaseClientSubmitter.cs @@ -50,64 +50,94 @@ namespace ArmoniK.DevelopmentKit.Client.Common.Submitter; /// Need to pass the child object Class Type /// [PublicAPI] -public class BaseClientSubmitter +// TODO: This should not be a public API. Public API should be defined in an interface. +public abstract class BaseClientSubmitter { + /// + /// The number of chunk to split the payloadsWithDependencies + /// + private readonly int chunkSubmitSize_; + + private readonly Properties properties_; + + /// + /// The channel pool to use for creating clients + /// + private ChannelPool? channelPool_; + /// /// Base Object for all Client submitter /// - /// Channel used to create grpc clients + /// Properties used to create grpc clients /// the logger factory to pass for root object + /// + /// /// The size of chunk to split the list of tasks - public BaseClientSubmitter(ChannelPool channelPool, - [CanBeNull] ILoggerFactory loggerFactory = null, - int chunkSubmitSize = 500) + protected BaseClientSubmitter(Properties properties, + ILoggerFactory loggerFactory, + TaskOptions taskOptions, + Session? session, + int chunkSubmitSize = 500) { - channelPool_ = channelPool; - Logger = loggerFactory?.CreateLogger(); + LoggerFactory = loggerFactory; + TaskOptions = taskOptions; + properties_ = properties; + Logger = loggerFactory.CreateLogger(); chunkSubmitSize_ = chunkSubmitSize; + SessionId = session ?? CreateSession(new[] + { + TaskOptions.PartitionId, + }); } + private ILoggerFactory LoggerFactory { get; } + /// /// Set or Get TaskOptions with inside MaxDuration, Priority, AppName, VersionName and AppNamespace /// - public TaskOptions TaskOptions { get; set; } + public TaskOptions TaskOptions { get; } /// /// Get SessionId object stored during the call of SubmitTask, SubmitSubTask, /// SubmitSubTaskWithDependencies or WaitForCompletion, WaitForSubTaskCompletion or GetResults /// - public Session SessionId { get; protected set; } - - -#pragma warning restore CS1591 + public Session SessionId { get; } /// /// The channel pool to use for creating clients /// - protected ChannelPool channelPool_; - - - /// - /// The number of chunk to split the payloadsWithDependencies - /// - private int chunkSubmitSize_; + public ChannelPool ChannelPool + => channelPool_ ??= ClientServiceConnector.ControlPlaneConnectionPool(properties_, + LoggerFactory); /// /// The logger to call the generate log in Seq /// - [CanBeNull] - protected ILogger Logger { get; set; } + protected ILogger Logger { get; } - /// - /// Service for interacting with results - /// - protected Results.ResultsClient ResultService { get; set; } + private Session CreateSession(IEnumerable partitionIds) + { + using var _ = Logger.LogFunction(); + Logger.LogDebug("Creating Session... "); + var createSessionRequest = new CreateSessionRequest + { + DefaultTaskOption = TaskOptions, + PartitionIds = + { + partitionIds, + }, + }; + var session = ChannelPool.WithChannel(channel => new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel).CreateSession(createSessionRequest)); + + Logger.LogDebug("Session Created {SessionId}", + SessionId); + return new Session + { + Id = session.SessionId, + }; + } - /// - /// Service for interacting with results - /// - protected Tasks.TasksClient TaskService { get; set; } /// /// Returns the status of the task @@ -127,15 +157,15 @@ public TaskStatus GetTaskStatus(string taskId) /// The list of taskIds /// public IEnumerable> GetTaskStatues(params string[] taskIds) - => channelPool_.WithChannel(channel => new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel).GetTaskStatus(new GetTaskStatusRequest - { - TaskIds = - { - taskIds, - }, - }) - .IdStatuses.Select(x => Tuple.Create(x.TaskId, - x.Status))); + => ChannelPool.WithChannel(channel => new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel).GetTaskStatus(new GetTaskStatusRequest + { + TaskIds = + { + taskIds, + }, + }) + .IdStatuses.Select(x => Tuple.Create(x.TaskId, + x.Status))); /// /// Return the taskOutput when error occurred @@ -143,11 +173,11 @@ public IEnumerable> GetTaskStatues(params string[] tas /// /// public Output GetTaskOutputInfo(string taskId) - => channelPool_.WithChannel(channel => new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel).TryGetTaskOutput(new TaskOutputRequest - { - TaskId = taskId, - Session = SessionId.Id, - })); + => ChannelPool.WithChannel(channel => new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel).TryGetTaskOutput(new TaskOutputRequest + { + TaskId = taskId, + Session = SessionId.Id, + })); /// /// The method to submit several tasks with dependencies tasks. This task will wait for @@ -164,11 +194,11 @@ public Output GetTaskOutputInfo(string taskId) [PublicAPI] public IEnumerable SubmitTasksWithDependencies(IEnumerable>> payloadsWithDependencies, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => payloadsWithDependencies.ToChunks(chunkSubmitSize_) .SelectMany(chunk => ChunkSubmitTasksWithDependencies(chunk, maxRetries, - taskOptions)); + taskOptions ?? TaskOptions)); /// /// The method to submit several tasks with dependencies tasks. This task will wait for @@ -187,22 +217,22 @@ public IEnumerable SubmitTasksWithDependencies(IEnumerable SubmitTasksWithDependencies(IEnumerable>> payloadsWithDependencies, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => payloadsWithDependencies.ToChunks(chunkSubmitSize_) .SelectMany(chunk => { - return ChunkSubmitTasksWithDependencies(chunk.Zip(CreateResultsMetadata(Enumerable.Range(0, - chunk.Length) - .Select(_ => Guid.NewGuid() - .ToString())) - .Values, - (subPayloadWithDependencies, - rid) => Tuple.Create(rid, - subPayloadWithDependencies.Item1, - subPayloadWithDependencies.Item2)) - .ToList(), + // Create the result metadata before sending the tasks. + var resultsMetadata = CreateResultsMetadata(Enumerable.Range(0, + chunk.Length) + .Select(_ => Guid.NewGuid() + .ToString())); + return ChunkSubmitTasksWithDependencies(chunk.Zip(resultsMetadata, + (payloadWithDependencies, + metadata) => Tuple.Create(metadata.Value, + payloadWithDependencies.Item1, + payloadWithDependencies.Item2)), maxRetries, - taskOptions); + taskOptions ?? TaskOptions); }); @@ -220,41 +250,42 @@ public IEnumerable SubmitTasksWithDependencies(IEnumerable ChunkSubmitTasksWithDependencies(IEnumerable>> payloadsWithDependencies, int maxRetries, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) { - using var _ = Logger?.LogFunction(); + using var _ = Logger.LogFunction(); + + var taskRequests = payloadsWithDependencies.Select(pwd => + { + var taskRequest = new TaskRequest + { + Payload = UnsafeByteOperations.UnsafeWrap(pwd.Item2), + }; + taskRequest.DataDependencies.AddRange(pwd.Item3); + taskRequest.ExpectedOutputKeys.Add(pwd.Item1); + return taskRequest; + }); for (var nbRetry = 0; nbRetry < maxRetries; nbRetry++) { try { - using var channel = channelPool_.GetChannel(); + using var channel = ChannelPool.GetChannel(); var submitterService = new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel); - //Multiple enumeration occurs on a retry var response = submitterService.CreateTasksAsync(SessionId.Id, taskOptions ?? TaskOptions, - payloadsWithDependencies.Select(pwd => - { - var taskRequest = new TaskRequest - { - Payload = UnsafeByteOperations.UnsafeWrap(pwd.Item2), - }; - taskRequest.DataDependencies - .AddRange(pwd.Item3 ?? Enumerable.Empty()); - taskRequest.ExpectedOutputKeys.Add(pwd.Item1); - return taskRequest; - })) + // multiple enumeration only occurs in case of failure + // ReSharper disable once PossibleMultipleEnumeration + taskRequests) .ConfigureAwait(false) .GetAwaiter() .GetResult(); return response.ResponseCase switch { - CreateTaskReply.ResponseOneofCase.CreationStatusList => response.CreationStatusList.CreationStatuses.Select(status => status.TaskInfo.TaskId) - .ToList(), - CreateTaskReply.ResponseOneofCase.None => throw new Exception("Issue with Server !"), - CreateTaskReply.ResponseOneofCase.Error => throw new Exception("Error while creating tasks !"), - _ => throw new ArgumentOutOfRangeException(), + CreateTaskReply.ResponseOneofCase.CreationStatusList => response.CreationStatusList.CreationStatuses.Select(status => status.TaskInfo.TaskId), + CreateTaskReply.ResponseOneofCase.None => throw new Exception("Issue with Server !"), + CreateTaskReply.ResponseOneofCase.Error => throw new Exception("Error while creating tasks !"), + _ => throw new InvalidOperationException(), }; } catch (Exception e) @@ -270,32 +301,32 @@ private IEnumerable ChunkSubmitTasksWithDependencies(IEnumerable 0) { - Logger?.LogWarning("{retry}/{maxRetries} nbRetry to submit batch of task", - nbRetry, - maxRetries); + Logger.LogWarning("{retry}/{maxRetries} nbRetry to submit batch of task", + nbRetry, + maxRetries); } } @@ -308,16 +339,20 @@ private IEnumerable ChunkSubmitTasksWithDependencies(IEnumerable /// The task taskId of the task to wait for /// - /// Option variable to set the number of retry (Default: 5) + /// Max number of retries for the underlying calls + /// Delay between retries public void WaitForTaskCompletion(string taskId, - int retry = 5) + int maxRetries = 5, + int delayMs = 20000) { - using var _ = Logger?.LogFunction(taskId); + using var _ = Logger.LogFunction(taskId); WaitForTasksCompletion(new[] { taskId, - }); + }, + maxRetries, + delayMs); } /// @@ -326,23 +361,27 @@ public void WaitForTaskCompletion(string taskId, /// /// List of taskIds /// + /// Max number of retries + /// [PublicAPI] - public void WaitForTasksCompletion(IEnumerable taskIds) + public void WaitForTasksCompletion(IEnumerable taskIds, + int maxRetries = 5, + int delayMs = 20000) { - using var _ = Logger?.LogFunction(); + using var _ = Logger.LogFunction(); - Retry.WhileException(5, - 2000, + Retry.WhileException(maxRetries, + delayMs, retry => { - using var channel = channelPool_.GetChannel(); + using var channel = ChannelPool.GetChannel(); var submitterService = new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel); if (retry > 1) { - Logger?.LogWarning("Try {try} for {funcName}", - retry, - nameof(submitterService.WaitForCompletion)); + Logger.LogWarning("Try {try} for {funcName}", + retry, + nameof(submitterService.WaitForCompletion)); } var __ = submitterService.WaitForCompletion(new WaitRequest @@ -393,12 +432,13 @@ public ResultStatusCollection GetResultStatus(IEnumerable taskIds, 2000, retry => { - using var channel = channelPool_.GetChannel(); + using var channel = ChannelPool.GetChannel(); var submitterService = new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel); - Logger?.LogDebug("Try {try} for {funcName}", - retry, - nameof(submitterService.GetResultStatus)); + Logger.LogDebug("Try {try} for {funcName}", + retry, + nameof(submitterService.GetResultStatus)); + // TODO: replace with submitterService.TryGetResultStream() => Issue # var resultStatusReply = submitterService.GetResultStatus(new GetResultStatusRequest { ResultIds = @@ -420,11 +460,8 @@ public ResultStatusCollection GetResultStatus(IEnumerable taskIds, foreach (var idStatusPair in idStatus) { - result2TaskDic.TryGetValue(idStatusPair.ResultId, - out var taskId); - var resData = new ResultStatusData(idStatusPair.ResultId, - taskId, + result2TaskDic[idStatusPair.ResultId], idStatusPair.Status); switch (idStatusPair.Status) @@ -447,14 +484,11 @@ public ResultStatusCollection GetResultStatus(IEnumerable taskIds, result2TaskDic.Remove(idStatusPair.ResultId); } - var resultStatusList = new ResultStatusCollection - { - IdsResultError = idsResultError, - IdsError = result2TaskDic.Values, - IdsReady = idsReady, - IdsNotReady = idsNotReady, - Canceled = missingTasks, - }; + var resultStatusList = new ResultStatusCollection(idsReady, + idsResultError, + result2TaskDic.Values.ToList(), + idsNotReady, + missingTasks.ToList()); return resultStatusList; } @@ -471,19 +505,19 @@ public ResultStatusCollection GetResultStatus(IEnumerable taskIds, { if (retry > 1) { - Logger?.LogWarning("Try {try} for {funcName}", - retry, - nameof(GetResultIds)); + Logger.LogWarning("Try {try} for {funcName}", + retry, + nameof(GetResultIds)); } - return channelPool_.WithChannel(channel => new Tasks.TasksClient(channel).GetResultIds(new GetResultIdsRequest - { - TaskId = - { - taskIds, - }, - }) - .TaskResults); + return ChannelPool.WithChannel(channel => new Tasks.TasksClient(channel).GetResultIds(new GetResultIdsRequest + { + TaskId = + { + taskIds, + }, + }) + .TaskResults); }, true, Logger, @@ -500,73 +534,73 @@ public ResultStatusCollection GetResultStatus(IEnumerable taskIds, public byte[] GetResult(string taskId, CancellationToken cancellationToken = default) { - using var _ = Logger?.LogFunction(taskId); - - var resultId = GetResultIds(new[] - { - taskId, - }) - .Single() - .ResultIds.Single(); - - - var resultRequest = new ResultRequest - { - ResultId = resultId, - Session = SessionId.Id, - }; - - Retry.WhileException(5, - 2000, - retry => - { - using var channel = channelPool_.GetChannel(); - var submitterService = new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel); - - Logger?.LogDebug("Try {try} for {funcName}", - retry, - nameof(submitterService.WaitForAvailability)); - var availabilityReply = submitterService.WaitForAvailability(resultRequest, - cancellationToken: cancellationToken); + using var _ = Logger.LogFunction(taskId); - switch (availabilityReply.TypeCase) + try + { + var resultId = GetResultIds(new[] + { + taskId, + }) + .Single() + .ResultIds.Single(); + + + var resultRequest = new ResultRequest + { + ResultId = resultId, + Session = SessionId.Id, + }; + + Retry.WhileException(5, + 2000, + retry => { - case AvailabilityReply.TypeOneofCase.None: - throw new Exception("Issue with Server !"); - case AvailabilityReply.TypeOneofCase.Ok: - break; - case AvailabilityReply.TypeOneofCase.Error: - throw new - ClientResultsException($"Result in Error - {resultId}\nMessage :\n{string.Join("Inner message:\n", availabilityReply.Error.Errors)}", - resultId); - case AvailabilityReply.TypeOneofCase.NotCompletedTask: - throw new DataException($"Result {resultId} was not yet completed"); - default: - throw new ArgumentOutOfRangeException(); - } - }, - true, - Logger, - typeof(IOException), - typeof(RpcException)); - - var res = Retry.WhileException(5, - 200, - retry => TryGetResultAsync(resultRequest, - cancellationToken) - .Result, - true, - Logger, - typeof(IOException), - typeof(RpcException)); - - if (res != null) + using var channel = ChannelPool.GetChannel(); + var submitterService = new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel); + + Logger.LogDebug("Try {try} for {funcName}", + retry, + nameof(submitterService.WaitForAvailability)); + // TODO: replace with submitterService.TryGetResultStream() => Issue # + var availabilityReply = submitterService.WaitForAvailability(resultRequest, + cancellationToken: cancellationToken); + + switch (availabilityReply.TypeCase) + { + case AvailabilityReply.TypeOneofCase.None: + throw new Exception("Issue with Server !"); + case AvailabilityReply.TypeOneofCase.Ok: + break; + case AvailabilityReply.TypeOneofCase.Error: + throw new + ClientResultsException($"Result in Error - {resultId}\nMessage :\n{string.Join("Inner message:\n", availabilityReply.Error.Errors)}", + resultId); + case AvailabilityReply.TypeOneofCase.NotCompletedTask: + throw new DataException($"Result {resultId} was not yet completed"); + default: + throw new InvalidOperationException(); + } + }, + true, + typeof(IOException), + typeof(RpcException)); + + return Retry.WhileException(5, + 200, + _ => TryGetResultAsync(resultRequest, + cancellationToken) + .Result, + true, + typeof(IOException), + typeof(RpcException))!; + } + catch (Exception ex) { - return res; + throw new ClientResultsException($"Cannot retrieve result for task : {taskId}", + ex, + taskId); } - - throw new ClientResultsException($"Cannot retrieve result for task : {taskId}", - taskId); } @@ -595,16 +629,17 @@ public IEnumerable> GetResults(IEnumerable taskIds /// /// Request specifying the result to fetch /// The token used to cancel the operation. - /// + /// Returns the result or byte[0] if there no result or null if task is not yet ready /// /// - public async Task TryGetResultAsync(ResultRequest resultRequest, - CancellationToken cancellationToken = default) + // TODO: return a compound type to avoid having a nullable that holds the information and return an empty array. + public async Task TryGetResultAsync(ResultRequest resultRequest, + CancellationToken cancellationToken = default) { List> chunks; int len; - using var channel = channelPool_.GetChannel(); + using var channel = ChannelPool.GetChannel(); var submitterService = new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel); { @@ -644,8 +679,7 @@ public async Task TryGetResultAsync(ResultRequest resultRequest, return null; default: - throw new ArgumentOutOfRangeException("Got a reply with an unexpected message type.", - (Exception)null); + throw new InvalidOperationException("Got a reply with an unexpected message type."); } } @@ -667,7 +701,6 @@ public async Task TryGetResultAsync(ResultRequest resultRequest, return res; } - /// /// Try to find the result of One task. If there no result, the function return byte[0] /// @@ -675,12 +708,28 @@ public async Task TryGetResultAsync(ResultRequest resultRequest, /// /// The optional cancellationToken /// Returns the result or byte[0] if there no result or null if task is not yet ready + // TODO: return a compound type to avoid having a nullable that holds the information and return an empty array. [PublicAPI] - public byte[] TryGetResult(string taskId, - bool checkOutput = true, - CancellationToken cancellationToken = default) + [Obsolete("Use version without the checkOutput parameter.")] + public byte[]? TryGetResult(string taskId, + bool checkOutput, + CancellationToken cancellationToken = default) + => TryGetResult(taskId, + cancellationToken); + + + /// + /// Try to find the result of One task. If there no result, the function return byte[0] + /// + /// The Id of the task + /// The optional cancellationToken + /// Returns the result or byte[0] if there no result or null if task is not yet ready + // TODO: return a compound type to avoid having a nullable that holds the information and return an empty array. + [PublicAPI] + public byte[]? TryGetResult(string taskId, + CancellationToken cancellationToken = default) { - using var _ = Logger?.LogFunction(taskId); + using var _ = Logger.LogFunction(taskId); var resultId = GetResultIds(new[] { taskId, @@ -700,9 +749,9 @@ public byte[] TryGetResult(string taskId, { if (retry > 1) { - Logger?.LogWarning("Try {try} for {funcName}", - retry, - "SubmitterService.TryGetResultAsync"); + Logger.LogWarning("Try {try} for {funcName}", + retry, + "SubmitterService.TryGetResultAsync"); } try @@ -742,8 +791,9 @@ public byte[] TryGetResult(string taskId, StatusCode: StatusCode.Aborted or StatusCode.Cancelled, }: - Logger?.LogError(rpcException, - rpcException.Message); + Logger.LogError(rpcException, + "Error while trying to get a result: {error}", + rpcException.Message); return null; default: throw; @@ -771,32 +821,36 @@ public IList> TryGetResults(IList resultIds) { if (resultStatus.IdsError.Any() || resultStatus.IdsResultError.Any()) { - var msg = - $"The missing result is in error or canceled. Please check log for more information on Armonik grid server list of taskIds in Error : [ {string.Join(", ", resultStatus.IdsResultError.Select(x => x.TaskId))}"; + var taskList = string.Join(", ", + resultStatus.IdsResultError.Select(x => x.TaskId)); if (resultStatus.IdsError.Any()) { if (resultStatus.IdsResultError.Any()) { - msg += ", "; + taskList += ", "; } - msg += $"{string.Join(", ", resultStatus.IdsError)}"; + taskList += string.Join(", ", + resultStatus.IdsError); } - msg += " ]\n"; - var taskIdInError = resultStatus.IdsError.Any() - ? resultStatus.IdsError.First() - : resultStatus.IdsResultError.First() - .TaskId; + ? resultStatus.IdsError[0] + : resultStatus.IdsResultError[0].TaskId; - msg += $"1st result id where the task which should create it is in error : {taskIdInError}"; + const string message = "The missing result is in error or canceled. " + + "Please check log for more information on Armonik grid server list of taskIds in Error: [{taskList}]\n" + + "1st result id where the task which should create it is in error : {taskIdInError}"; - Logger?.LogError(msg); + Logger.LogError(message, + taskList, + taskIdInError); - throw new ClientResultsException(msg, - (resultStatus.IdsError ?? Enumerable.Empty()).Concat(resultStatus.IdsResultError.Select(x => x.TaskId))); + throw new + ClientResultsException($"The missing result is in error or canceled. Please check log for more information on Armonik grid server list of taskIds in Error: [{taskList}]" + + $"1st result id where the task which should create it is in error : {taskIdInError}", + resultStatus.IdsError.ToArray()); } } @@ -813,7 +867,8 @@ public IList> TryGetResults(IList resultIds) : new Tuple(resultStatusData.TaskId, res); }) - .Where(el => el != null) + .Where(tuple => tuple is not null) + .Select(tuple => tuple!) .ToList(); } @@ -824,18 +879,17 @@ public IList> TryGetResults(IList resultIds) /// Dictionary where each result name is associated with its result id [PublicAPI] public Dictionary CreateResultsMetadata(IEnumerable resultNames) - => channelPool_.WithChannel(c => new Results.ResultsClient(c).CreateResultsMetaData(new CreateResultsMetaDataRequest - { - SessionId = SessionId.Id, - Results = - { - resultNames.Select(name - => new CreateResultsMetaDataRequest.Types.ResultCreate - { - Name = name, - }), - }, - })) - .Results.ToDictionary(r => r.Name, - r => r.ResultId); + => ChannelPool.WithChannel(c => new Results.ResultsClient(c).CreateResultsMetaData(new CreateResultsMetaDataRequest + { + SessionId = SessionId.Id, + Results = + { + resultNames.Select(name => new CreateResultsMetaDataRequest.Types.ResultCreate + { + Name = name, + }), + }, + })) + .Results.ToDictionary(r => r.Name, + r => r.ResultId); } diff --git a/Client/src/Common/Submitter/ChannelPool.cs b/Client/src/Common/Submitter/ChannelPool.cs index ea8f07a8..9981a83d 100644 --- a/Client/src/Common/Submitter/ChannelPool.cs +++ b/Client/src/Common/Submitter/ChannelPool.cs @@ -19,8 +19,6 @@ using Grpc.Core; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; #if NET5_0_OR_GREATER using Grpc.Net.Client; @@ -35,8 +33,7 @@ public sealed class ChannelPool { private readonly Func channelFactory_; - [CanBeNull] - private readonly ILogger logger_; + private readonly ILogger? logger_; private readonly ConcurrentBag pool_; @@ -45,8 +42,8 @@ public sealed class ChannelPool /// /// Function used to create new channels /// loggerFactory used to instantiate a logger for the pool - public ChannelPool(Func channelFactory, - [CanBeNull] ILoggerFactory loggerFactory = null) + public ChannelPool(Func channelFactory, + ILoggerFactory? loggerFactory = null) { channelFactory_ = channelFactory; pool_ = new ConcurrentBag(); @@ -168,17 +165,7 @@ public ChannelGuard GetChannel() public T WithChannel(Func f) { using var channel = GetChannel(); - return f(channel.Channel); - } - - /// - /// Call f with an acquired channel - /// - /// Function to be called - public void WithChannel(Action f) - { - using var channel = GetChannel(); - f(channel.Channel); + return f(channel); } /// @@ -189,7 +176,7 @@ public sealed class ChannelGuard : IDisposable /// /// Channel that is used by nobody else /// - public readonly ChannelBase Channel; + private readonly ChannelBase channel_; private readonly ChannelPool pool_; @@ -199,13 +186,13 @@ public sealed class ChannelGuard : IDisposable /// public ChannelGuard(ChannelPool channelPool) { - pool_ = channelPool; - Channel = channelPool.AcquireChannel(); + pool_ = channelPool; + channel_ = channelPool.AcquireChannel(); } /// public void Dispose() - => pool_.ReleaseChannel(Channel); + => pool_.ReleaseChannel(channel_); /// /// Implicit convert a ChannelGuard into a ChannelBase @@ -213,6 +200,6 @@ public void Dispose() /// ChannelGuard /// ChannelBase public static implicit operator ChannelBase(ChannelGuard guard) - => guard.Channel; + => guard.channel_; } } diff --git a/Client/src/Common/Submitter/ClientServiceConnector.cs b/Client/src/Common/Submitter/ClientServiceConnector.cs index c266c0b9..bc5be60d 100644 --- a/Client/src/Common/Submitter/ClientServiceConnector.cs +++ b/Client/src/Common/Submitter/ClientServiceConnector.cs @@ -19,8 +19,6 @@ using ArmoniK.Api.Client.Options; using ArmoniK.Api.Client.Submitter; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; namespace ArmoniK.DevelopmentKit.Client.Common.Submitter; @@ -37,12 +35,12 @@ public class ClientServiceConnector /// Configuration Properties /// Optional logger factory /// The connection pool - public static ChannelPool ControlPlaneConnectionPool(Properties properties, - [CanBeNull] ILoggerFactory loggerFactory = null) + public static ChannelPool ControlPlaneConnectionPool(Properties properties, + ILoggerFactory? loggerFactory = null) { var options = new GrpcClient { - AllowUnsafeConnection = !properties.ConfSSLValidation, + AllowUnsafeConnection = !properties.ConfSslValidation, CaCert = properties.CaCertFilePem, CertP12 = properties.ClientP12File, CertPem = properties.ClientCertFilePem, diff --git a/Client/src/Common/Submitter/ISubmitterService.cs b/Client/src/Common/Submitter/ISubmitterService.cs index 1f98f6ad..03e4eaa7 100644 --- a/Client/src/Common/Submitter/ISubmitterService.cs +++ b/Client/src/Common/Submitter/ISubmitterService.cs @@ -14,14 +14,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Api.gRPC.V1; + +using JetBrains.Annotations; namespace ArmoniK.DevelopmentKit.Client.Common.Submitter; /// /// Common Interface for Submitter services. /// -public interface ISubmitterService +[PublicAPI] +public interface ISubmitterService : IDisposable { /// /// The Id of the current session @@ -29,15 +38,25 @@ public interface ISubmitterService string SessionId { get; } /// - /// The method submit will execute task asynchronously on the server + /// The method submits a list of task with a list of arguments for each task which will be serialized into a byte[] for + /// each call + /// MethodName(byte[] argument) /// /// The name of the method inside the service - /// A list of objects that can be passed in parameters of the function + /// A list of parameters that can be passed in parameters of the each call of function /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error - /// Return the taskId string - string Submit(string methodName, - object[] arguments, - IServiceInvocationHandler handler); + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// + /// Return the list of created taskIds + IEnumerable Submit(string methodName, + IEnumerable arguments, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null); + /// /// The method submits a list of task with a list of arguments for each task which will be serialized into a byte[] for @@ -45,10 +64,122 @@ string Submit(string methodName, /// MethodName(byte[] argument) /// /// The name of the method inside the service - /// A list of parameters that can be passed in parameters of the each call of function + /// List of serialized arguments that will already serialize for MethodName. /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// /// Return the list of created taskIds IEnumerable Submit(string methodName, - IEnumerable arguments, - IServiceInvocationHandler handler); + IEnumerable arguments, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null); + + /// + /// The method submit with one serialized argument that will send as byte[] MethodName(byte[] argument). + /// + /// The name of the method inside the service + /// One serialized argument that will already serialize for MethodName. + /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// + /// The cancellation token to set to cancel the async task + /// Return the taskId string + public Task SubmitAsync(string methodName, + object[] argument, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null, + CancellationToken token = default); + + /// + /// The method submit with one serialized argument that will send as byte[] MethodName(byte[] argument). + /// + /// The name of the method inside the service + /// One serialized argument that will already serialize for MethodName. + /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// + /// The cancellation token to set to cancel the async task + /// Return the taskId string + public Task SubmitAsync(string methodName, + byte[] argument, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null, + CancellationToken token = default); +} + +/// +/// Provide extension methods for ISubmitterService +/// +[PublicAPI] +public static class SubmitterServiceExt +{ + /// + /// The method submit will execute task asynchronously on the server + /// + /// the ISubmitterService extended + /// The name of the method inside the service + /// A list of objects that can be passed in parameters of the function + /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// + /// Return the taskId string + public static string Submit(this ISubmitterService service, + string methodName, + object[] arguments, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null) + => service.Submit(methodName, + new[] + { + arguments, + }, + handler, + maxRetries, + taskOptions) + .Single(); + + /// + /// The method submit will execute task asynchronously on the server + /// + /// the ISubmitterService extended + /// The name of the method inside the service + /// List of serialized arguments that will already serialize for MethodName. + /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// + /// Return the taskId string + public static string Submit(this ISubmitterService service, + string methodName, + byte[] arguments, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null) + => service.Submit(methodName, + new[] + { + arguments, + }, + handler, + maxRetries, + taskOptions) + .Single(); } diff --git a/Client/src/Common/Submitter/RequestTaskMap.cs b/Client/src/Common/Submitter/RequestTaskMap.cs index fd14f196..1cddf015 100644 --- a/Client/src/Common/Submitter/RequestTaskMap.cs +++ b/Client/src/Common/Submitter/RequestTaskMap.cs @@ -24,7 +24,7 @@ namespace ArmoniK.DevelopmentKit.Client.Common.Submitter; /// -/// The class to map submitId to taskId when submit is done asynchronously +/// The class to map submissionId to taskId when submit is done asynchronously /// public class RequestTaskMap { @@ -32,27 +32,27 @@ public class RequestTaskMap private readonly ConcurrentDictionary> dictionary_ = new(); /// - /// Push the SubmitId and taskId in the concurrentDictionary + /// Push the submissionId and taskId in the concurrentDictionary /// - /// The submit Id push during the submission + /// The submit Id push during the submission /// the taskId was given by the control Plane - public void PutResponse(Guid SubmitId, + public void PutResponse(Guid submitId, string taskId) - => dictionary_[SubmitId] = taskId; + => dictionary_[submitId] = taskId; /// - /// Get the correct taskId based on the SubmitId + /// Get the correct taskId based on the submissionId /// - /// The submit Id push during the submission + /// The submit Id push during the submission /// the async taskId - public async Task GetResponseAsync(Guid submitId) + public async Task GetResponseAsync(Guid submissionId) { - while (!dictionary_.ContainsKey(submitId)) + while (!dictionary_.ContainsKey(submissionId)) { await Task.Delay(WaitTime); } - return dictionary_[submitId] + return dictionary_[submissionId] .IfRight(e => { throw e; @@ -63,12 +63,12 @@ public async Task GetResponseAsync(Guid submitId) /// /// Notice user that there was at least one error during the submission of buffer /// - /// + /// /// exception occurring the submission - public void BufferFailures(IEnumerable submitIds, + public void BufferFailures(IEnumerable submissionIds, Exception exception) { - foreach (var submitId in submitIds) + foreach (var submitId in submissionIds) { dictionary_[submitId] = exception; } diff --git a/Client/src/Symphony/ArmoniK.DevelopmentKit.Client.Symphony.csproj b/Client/src/Symphony/ArmoniK.DevelopmentKit.Client.Symphony.csproj index 16854cd3..72614760 100644 --- a/Client/src/Symphony/ArmoniK.DevelopmentKit.Client.Symphony.csproj +++ b/Client/src/Symphony/ArmoniK.DevelopmentKit.Client.Symphony.csproj @@ -1,7 +1,7 @@ - net472;net48;net5.0;net6.0 + netstandard2.0 Library True true @@ -9,7 +9,6 @@ - diff --git a/Client/src/Symphony/ArmonikSymphonyClient.cs b/Client/src/Symphony/ArmonikSymphonyClient.cs index 3d7cc3cb..38bdeeef 100644 --- a/Client/src/Symphony/ArmonikSymphonyClient.cs +++ b/Client/src/Symphony/ArmonikSymphonyClient.cs @@ -73,29 +73,36 @@ public ArmonikSymphonyClient(IConfiguration configuration, /// /// Optional parameter to set TaskOptions during the Session creation /// Returns the SessionService to submit, wait or get result - public SessionService CreateSession(TaskOptions taskOptions = null) + public SessionService CreateSession(TaskOptions? taskOptions = null) { ControlPlaneConnection(); - return new SessionService(GrpcPool, + + var properties = new Properties(Configuration, + taskOptions); + + return new SessionService(properties, LoggerFactory, - taskOptions); + taskOptions ?? SessionService.InitializeDefaultTaskOptions()); } /// /// Open the session already created to submit task /// /// The sessionId string which will opened - /// the customer taskOptions send to the server by the client + /// the customer taskOptions send to the server by the client /// Returns the SessionService to submit, wait or get result - public SessionService OpenSession(Session sessionId, - TaskOptions clientOptions = null) + public SessionService OpenSession(Session sessionId, + TaskOptions? taskOptions = null) { ControlPlaneConnection(); - return new SessionService(GrpcPool, + var properties = new Properties(Configuration, + taskOptions); + + return new SessionService(properties, LoggerFactory, - clientOptions ?? SessionService.InitializeDefaultTaskOptions(), + taskOptions ?? SessionService.InitializeDefaultTaskOptions(), sessionId); } diff --git a/Client/src/Symphony/SessionService.cs b/Client/src/Symphony/SessionService.cs index 89e8e13e..4053b210 100644 --- a/Client/src/Symphony/SessionService.cs +++ b/Client/src/Symphony/SessionService.cs @@ -18,16 +18,13 @@ using System.Collections.Generic; using System.Linq; -using ArmoniK.Api.Common.Utils; using ArmoniK.Api.gRPC.V1; -using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Common; using Google.Protobuf.WellKnownTypes; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; namespace ArmoniK.DevelopmentKit.Client.Symphony; @@ -43,34 +40,28 @@ public class SessionService : BaseClientSubmitter /// Ctor to instantiate a new SessionService /// This is an object to send task or get Results from a session /// - public SessionService(ChannelPool channelPool, - [CanBeNull] ILoggerFactory loggerFactory = null, - [CanBeNull] TaskOptions taskOptions = null, - [CanBeNull] Session session = null) - : base(channelPool, - loggerFactory) + public SessionService(Properties properties, + ILoggerFactory loggerFactory, + TaskOptions? taskOptions = null, + Session? session = null) + : base(properties, + loggerFactory, + taskOptions ?? InitializeDefaultTaskOptions(), + session) { - TaskOptions = taskOptions ?? InitializeDefaultTaskOptions(); - - Logger?.LogDebug("Creating Session... "); - - SessionId = session ?? CreateSession(new List - { - TaskOptions.PartitionId, - }); - - Logger?.LogDebug($"Session Created {SessionId}"); } /// Returns a string that represents the current object. /// A string that represents the current object. public override string ToString() - => SessionId?.Id ?? "Session_Not_ready"; + => SessionId.Id ?? "Session_Not_ready"; /// /// Default task options /// /// + // TODO: mark with [PublicApi] ? + // ReSharper disable once MemberCanBePrivate.Global public static TaskOptions InitializeDefaultTaskOptions() => new() { @@ -86,39 +77,6 @@ public static TaskOptions InitializeDefaultTaskOptions() ApplicationNamespace = "ArmoniK.Samples.Symphony.Packages", }; - private Session CreateSession(IEnumerable partitionIds) - { - using var _ = Logger?.LogFunction(); - var createSessionRequest = new CreateSessionRequest - { - DefaultTaskOption = TaskOptions, - PartitionIds = - { - partitionIds, - }, - }; - var session = channelPool_.WithChannel(channel => new Submitter.SubmitterClient(channel).CreateSession(createSessionRequest)); - - return new Session - { - Id = session.SessionId, - }; - } - - /// - /// Set connection to an already opened Session - /// - /// SessionId previously opened - public void OpenSession(Session session) - { - if (SessionId == null) - { - Logger?.LogDebug($"Open Session {session.Id}"); - } - - SessionId = session; - } - /// /// User method to submit task from the client /// Need a client Service. In case of ServiceContainer @@ -134,9 +92,9 @@ public void OpenSession(Session session) /// public IEnumerable SubmitTasks(IEnumerable payloads, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => SubmitTasksWithDependencies(payloads.Select(payload => new Tuple>(payload, - null)), + Array.Empty())), maxRetries, taskOptions); @@ -151,9 +109,9 @@ public IEnumerable SubmitTasks(IEnumerable payloads, /// TaskOptions argument to override default taskOptions in Session. /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// - public string SubmitTask(byte[] payload, - int maxRetries = 5, - TaskOptions taskOptions = null) + public string SubmitTask(byte[] payload, + int maxRetries = 5, + TaskOptions? taskOptions = null) => SubmitTasks(new[] { payload, @@ -175,10 +133,12 @@ public string SubmitTask(byte[] payload, /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// /// return the taskId of the created task + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedMember.Global public string SubmitTaskWithDependencies(byte[] payload, IList dependencies, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => SubmitTasksWithDependencies(new[] { Tuple.Create(payload, diff --git a/Client/src/Unified/ArmoniK.DevelopmentKit.Client.Unified.csproj b/Client/src/Unified/ArmoniK.DevelopmentKit.Client.Unified.csproj index 83eefd00..2db5a87d 100644 --- a/Client/src/Unified/ArmoniK.DevelopmentKit.Client.Unified.csproj +++ b/Client/src/Unified/ArmoniK.DevelopmentKit.Client.Unified.csproj @@ -1,16 +1,14 @@ - net472;net48;net5.0;net6.0 + netstandard2.0 Library True true - - diff --git a/Client/src/Unified/Factory/ServiceFactory.cs b/Client/src/Unified/Factory/ServiceFactory.cs index ee47a625..244817b8 100644 --- a/Client/src/Unified/Factory/ServiceFactory.cs +++ b/Client/src/Unified/Factory/ServiceFactory.cs @@ -15,13 +15,13 @@ // limitations under the License. using ArmoniK.DevelopmentKit.Client.Common; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Services.Admin; using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace ArmoniK.DevelopmentKit.Client.Unified.Factory; @@ -37,10 +37,10 @@ public class ServiceFactory /// Properties for the service containing IConfiguration and TaskOptions /// Logger factory to create loggers for service /// returns the new instantiated service - public static Service CreateService(Properties props, - [CanBeNull] ILoggerFactory loggerFactory = null) - => new(props, - loggerFactory); + public static ISubmitterService CreateService(Properties props, + ILoggerFactory? loggerFactory = null) + => new Service(props, + loggerFactory); /// /// Method to get the ServiceAdmin @@ -48,8 +48,8 @@ public static Service CreateService(Properties props, /// /// Logger factory to create loggers for service /// - public static ServiceAdmin GetServiceAdmin(Properties props, - [CanBeNull] ILoggerFactory loggerFactory = null) + public static ServiceAdmin GetServiceAdmin(Properties props, + ILoggerFactory? loggerFactory = null) => new(props, - loggerFactory); + loggerFactory ?? new NullLoggerFactory()); } diff --git a/Client/src/Unified/Factory/SessionServiceFactory.cs b/Client/src/Unified/Factory/SessionServiceFactory.cs index 9ed67260..ab8a2844 100644 --- a/Client/src/Unified/Factory/SessionServiceFactory.cs +++ b/Client/src/Unified/Factory/SessionServiceFactory.cs @@ -23,9 +23,8 @@ using Google.Protobuf.WellKnownTypes; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace ArmoniK.DevelopmentKit.Client.Unified.Factory; @@ -44,16 +43,15 @@ public class SessionServiceFactory /// The ctor with IConfiguration and optional TaskOptions /// /// The factory to create the logger for clientService - public SessionServiceFactory([CanBeNull] ILoggerFactory loggerFactory = null) + public SessionServiceFactory(ILoggerFactory? loggerFactory = null) { - LoggerFactory = loggerFactory; - Logger = loggerFactory?.CreateLogger(); + LoggerFactory = loggerFactory ?? new NullLoggerFactory(); + Logger = LoggerFactory.CreateLogger(); } - [CanBeNull] private ILogger Logger { get; } - private ChannelPool GrpcPool { get; set; } + private ChannelPool? GrpcPool { get; set; } private ILoggerFactory LoggerFactory { get; } @@ -69,7 +67,7 @@ public SessionService CreateSession(Properties properties) Logger?.LogDebug("Creating Session... "); - return new SessionService(GrpcPool, + return new SessionService(properties, LoggerFactory, properties.TaskOptions); } @@ -92,13 +90,13 @@ private void ControlPlaneConnection(Properties properties) /// The properties setting for the session /// SessionId previously opened /// - public SessionService OpenSession(Properties properties, - string sessionId, - TaskOptions clientOptions = null) + public SessionService OpenSession(Properties properties, + string sessionId, + TaskOptions? clientOptions = null) { ControlPlaneConnection(properties); - return new SessionService(GrpcPool, + return new SessionService(properties, LoggerFactory, clientOptions, new Session diff --git a/Client/src/Unified/Services/Admin/AdminMonitoringService.cs b/Client/src/Unified/Services/Admin/AdminMonitoringService.cs index a6d34112..ff40719a 100644 --- a/Client/src/Unified/Services/Admin/AdminMonitoringService.cs +++ b/Client/src/Unified/Services/Admin/AdminMonitoringService.cs @@ -22,8 +22,6 @@ using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.DevelopmentKit.Client.Common.Submitter; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; namespace ArmoniK.DevelopmentKit.Client.Unified.Services.Admin; @@ -40,15 +38,14 @@ public class AdminMonitoringService /// /// The entry point to the control plane /// The factory logger to create logger - public AdminMonitoringService(ChannelPool channelPool, - [CanBeNull] ILoggerFactory loggerFactory = null) + public AdminMonitoringService(ChannelPool channelPool, + ILoggerFactory? loggerFactory = null) { Logger = loggerFactory?.CreateLogger(); channelPool_ = channelPool; } - [CanBeNull] - private ILogger Logger { get; } + private ILogger? Logger { get; } /// diff --git a/Client/src/Unified/Services/Admin/ServiceAdmin.cs b/Client/src/Unified/Services/Admin/ServiceAdmin.cs index 993a4a63..ead0f9e9 100644 --- a/Client/src/Unified/Services/Admin/ServiceAdmin.cs +++ b/Client/src/Unified/Services/Admin/ServiceAdmin.cs @@ -34,8 +34,7 @@ public class ServiceAdmin : AbstractClientService /// public ServiceAdmin(Properties properties, ILoggerFactory loggerFactory) - : base(properties, - loggerFactory) + : base(loggerFactory) { SessionServiceFactory = new SessionServiceFactory(LoggerFactory); diff --git a/Client/src/Unified/Services/Common/AbstractClientService.cs b/Client/src/Unified/Services/Common/AbstractClientService.cs index e97947a5..4221fb5b 100644 --- a/Client/src/Unified/Services/Common/AbstractClientService.cs +++ b/Client/src/Unified/Services/Common/AbstractClientService.cs @@ -20,9 +20,8 @@ using ArmoniK.DevelopmentKit.Client.Common; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace ArmoniK.DevelopmentKit.Client.Unified.Services.Common; @@ -34,12 +33,10 @@ public abstract class AbstractClientService : IDisposable /// /// The default constructor with properties information /// - /// /// - public AbstractClientService(Properties properties, - [CanBeNull] ILoggerFactory loggerFactory = null) + protected AbstractClientService(ILoggerFactory? loggerFactory = null) { - LoggerFactory = loggerFactory; + LoggerFactory = loggerFactory ?? new NullLoggerFactory(); ResultHandlerDictionary = new ConcurrentDictionary(); } @@ -55,13 +52,12 @@ public IReadOnlyCollection CurrentlyHandledTaskIds /// /// The result dictionary to return result /// - protected ConcurrentDictionary ResultHandlerDictionary { get; set; } + protected ConcurrentDictionary ResultHandlerDictionary { get; } /// /// The properties to get LoggerFactory or to override it /// - [CanBeNull] - protected ILoggerFactory LoggerFactory { get; set; } + protected ILoggerFactory LoggerFactory { get; } /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. public abstract void Dispose(); diff --git a/Client/src/Unified/Services/SessionService.cs b/Client/src/Unified/Services/SessionService.cs index f8284677..14417e6c 100644 --- a/Client/src/Unified/Services/SessionService.cs +++ b/Client/src/Unified/Services/SessionService.cs @@ -19,16 +19,13 @@ using System.Linq; using System.Threading; -using ArmoniK.Api.Common.Utils; using ArmoniK.Api.gRPC.V1; -using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Common; using Google.Protobuf.WellKnownTypes; -using JetBrains.Annotations; - using Microsoft.Extensions.Logging; namespace ArmoniK.DevelopmentKit.Client.Unified.Services; @@ -44,47 +41,28 @@ public class SessionService : BaseClientSubmitter /// Ctor to instantiate a new SessionService /// This is an object to send task or get Results from a session /// - public SessionService(ChannelPool channelPool, - [CanBeNull] ILoggerFactory loggerFactory = null, - [CanBeNull] TaskOptions taskOptions = null, - [CanBeNull] Session session = null) - : base(channelPool, - loggerFactory) + public SessionService(Properties properties, + ILoggerFactory loggerFactory, + TaskOptions? taskOptions = null, + Session? session = null) + : base(properties, + loggerFactory, + taskOptions ?? InitializeDefaultTaskOptions(), + session) { - TaskOptions = taskOptions ?? InitializeDefaultTaskOptions(); - - Logger?.LogDebug("Creating Session... "); - - SessionId = session ?? CreateSession(new List - { - taskOptions.PartitionId, - }); - - Logger?.LogDebug($"Session Created {SessionId}"); } - /// - /// Return the Grpc channel pool - /// - public ChannelPool ChannelPool - => channelPool_; - /// Returns a string that represents the current object. - /// A string that represents the current object. + /// public override string ToString() - { - if (SessionId?.Id != null) - { - return SessionId?.Id; - } - - return "Session_Not_ready"; - } + => SessionId.Id ?? "Session_Not_ready"; /// /// Supply a default TaskOptions /// /// A default TaskOptions object + // TODO: mark with [PublicApi] ? + // ReSharper disable once MemberCanBePrivate.Global public static TaskOptions InitializeDefaultTaskOptions() { TaskOptions taskOptions = new() @@ -105,39 +83,6 @@ public static TaskOptions InitializeDefaultTaskOptions() return taskOptions; } - private Session CreateSession(IEnumerable partitionIds) - { - using var _ = Logger?.LogFunction(); - var createSessionRequest = new CreateSessionRequest - { - DefaultTaskOption = TaskOptions, - PartitionIds = - { - partitionIds, - }, - }; - var session = channelPool_.WithChannel(channel => new Api.gRPC.V1.Submitter.Submitter.SubmitterClient(channel).CreateSession(createSessionRequest)); - - return new Session - { - Id = session.SessionId, - }; - } - - /// - /// Set connection to an already opened Session - /// - /// SessionId previously opened - public void OpenSession(Session session) - { - if (SessionId == null) - { - Logger?.LogDebug($"Open Session {session.Id}"); - } - - SessionId = session; - } - /// /// User method to submit task from the client /// Need a client Service. In case of ServiceContainer @@ -153,9 +98,9 @@ public void OpenSession(Session session) /// public IEnumerable SubmitTasks(IEnumerable payloads, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => SubmitTasksWithDependencies(payloads.Select(payload => new Tuple>(payload, - null)), + Array.Empty())), maxRetries, taskOptions); @@ -171,11 +116,12 @@ public IEnumerable SubmitTasks(IEnumerable payloads, /// TaskOptions argument to override default taskOptions in Session. /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// - public string SubmitTask(byte[] payload, - int waitTimeBeforeNextSubmit = 2, - int maxRetries = 5, - TaskOptions taskOptions = null) + public string SubmitTask(byte[] payload, + int waitTimeBeforeNextSubmit = 2, + int maxRetries = 5, + TaskOptions? taskOptions = null) { + // TODO: wtf? Thread.Sleep(waitTimeBeforeNextSubmit); // Twice the keep alive return SubmitTasks(new[] { @@ -199,10 +145,12 @@ public string SubmitTask(byte[] payload, /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// /// return the taskId of the created task + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedMember.Global public string SubmitTaskWithDependencies(byte[] payload, IList dependencies, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => SubmitTasksWithDependencies(new[] { Tuple.Create(payload, @@ -211,5 +159,4 @@ public string SubmitTaskWithDependencies(byte[] payload, maxRetries, taskOptions) .Single(); -#pragma warning restore CS1591 } diff --git a/Client/src/Unified/Services/Submitter/BatchUntilInactiveBlock.cs b/Client/src/Unified/Services/Submitter/BatchUntilInactiveBlock.cs index 2b09af04..7ade944d 100644 --- a/Client/src/Unified/Services/Submitter/BatchUntilInactiveBlock.cs +++ b/Client/src/Unified/Services/Submitter/BatchUntilInactiveBlock.cs @@ -20,8 +20,6 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; -using JetBrains.Annotations; - namespace ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; /// @@ -47,9 +45,9 @@ public class BatchUntilInactiveBlock : IPropagatorBlock, IReceivableS /// Options to configure message. /// https://learn.microsoft.com/fr-fr/dotnet/api/system.threading.tasks.dataflow.executiondataflowblockoptions?view=net-6.0 /// - public BatchUntilInactiveBlock(int bufferRequestsSize, - TimeSpan timeout, - [CanBeNull] ExecutionDataflowBlockOptions executionDataFlowBlockOptions = null) + public BatchUntilInactiveBlock(int bufferRequestsSize, + TimeSpan timeout, + ExecutionDataflowBlockOptions? executionDataFlowBlockOptions = null) { executionDataFlowBlockOptions_ = executionDataFlowBlockOptions ?? new ExecutionDataflowBlockOptions { diff --git a/Client/src/Unified/Services/Submitter/BlockRequest.cs b/Client/src/Unified/Services/Submitter/BlockRequest.cs index dfc4091e..aa47ef97 100644 --- a/Client/src/Unified/Services/Submitter/BlockRequest.cs +++ b/Client/src/Unified/Services/Submitter/BlockRequest.cs @@ -21,16 +21,13 @@ using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Common; -using JetBrains.Annotations; - namespace ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; internal class BlockRequest { public IServiceInvocationHandler Handler; - [CanBeNull] - public ArmonikPayload Payload { get; set; } + public ArmonikPayload? Payload { get; set; } public SemaphoreSlim Lock { get; set; } public Guid SubmitId { get; set; } diff --git a/Client/src/Unified/Services/Submitter/Service.cs b/Client/src/Unified/Services/Submitter/Service.cs index 45bd96bd..812fd442 100644 --- a/Client/src/Unified/Services/Submitter/Service.cs +++ b/Client/src/Unified/Services/Submitter/Service.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using ArmoniK.DevelopmentKit.Client.Common.Status; using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Factory; using ArmoniK.DevelopmentKit.Client.Unified.Services.Common; @@ -50,57 +51,20 @@ namespace ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; [MarkDownDoc] public class Service : AbstractClientService, ISubmitterService { - private const int MaxRetries = 10; - - // *** you need some mechanism to map types to fields - private static readonly IDictionary StatusCodesLookUp = new List> - { - Tuple.Create(TaskStatus.Submitted, - ArmonikStatusCode.ResultNotReady), - Tuple.Create(TaskStatus.Timeout, - ArmonikStatusCode.TaskTimeout), - Tuple.Create(TaskStatus.Cancelled, - ArmonikStatusCode.TaskCancelled), - Tuple.Create(TaskStatus.Cancelling, - ArmonikStatusCode.TaskCancelled), - Tuple.Create(TaskStatus.Error, - ArmonikStatusCode.TaskFailed), - Tuple.Create(TaskStatus.Processing, - ArmonikStatusCode.ResultNotReady), - Tuple.Create(TaskStatus.Dispatched, - ArmonikStatusCode.ResultNotReady), - Tuple.Create(TaskStatus.Completed, - ArmonikStatusCode.ResultReady), - Tuple.Create(TaskStatus.Creating, - ArmonikStatusCode.ResultNotReady), - Tuple.Create(TaskStatus.Unspecified, - ArmonikStatusCode.TaskFailed), - Tuple.Create(TaskStatus.Processed, - ArmonikStatusCode.ResultReady), - }.ToDictionary(k => k.Item1, - v => v.Item2); - private readonly RequestTaskMap requestTaskMap_ = new(); private readonly SemaphoreSlim semaphoreSlim_; - /// - /// Property Get the SessionId - /// - [PublicAPI] - public SessionService SessionService; - /// /// The default constructor to open connection with the control plane /// and create the session to ArmoniK /// /// The properties containing TaskOptions and information to communicate with Control plane and /// - public Service(Properties properties, - [CanBeNull] ILoggerFactory loggerFactory = null) - : base(properties, - loggerFactory) + public Service(Properties properties, + ILoggerFactory? loggerFactory = null) + : base(loggerFactory) { var timeOutSending = properties.TimeTriggerBuffer ?? TimeSpan.FromSeconds(1); @@ -112,17 +76,14 @@ public Service(Properties properties, SessionService = SessionServiceFactory.CreateSession(properties); - ProtoSerializer = new ProtoSerializer(); - CancellationResultTaskSource = new CancellationTokenSource(); - CancellationQueueTaskSource = new CancellationTokenSource(); HandlerResponse = Task.Run(ResultTask, CancellationResultTaskSource.Token); - Logger = LoggerFactory?.CreateLogger(); - Logger?.BeginPropertyScope(("SessionId", SessionService.SessionId), - ("Class", "Service")); + Logger = LoggerFactory.CreateLogger(); + Logger.BeginPropertyScope(("SessionId", SessionService.SessionId), + ("Class", "Service")); BufferSubmit = new BatchUntilInactiveBlock(maxTasksPerBuffer, timeOutSending, @@ -143,8 +104,8 @@ public Service(Properties properties, return; } - Logger?.LogInformation("Submitting buffer of {count} task...", - blockRequestList.Count); + Logger.LogInformation("Submitting buffer of {count} task...", + blockRequestList.Count); var query = blockRequestList.GroupBy(blockRequest => blockRequest.TaskOptions); foreach (var groupBlockRequest in query) @@ -152,14 +113,14 @@ public Service(Properties properties, var maxRetries = groupBlockRequest.First() .MaxRetries; //Generate resultId - var results_ids = SessionService.CreateResultsMetadata(groupBlockRequest.Select(_ => Guid.NewGuid() - .ToString())) - .Values.ToList(); + var resultsIds = SessionService.CreateResultsMetadata(groupBlockRequest.Select(_ => Guid.NewGuid() + .ToString())) + .Values.ToList(); foreach (var (request, index) in groupBlockRequest.Select((r, i) => (r, i))) { - request.ResultId = results_ids[index]; + request.ResultId = resultsIds[index]; } var currentBackoff = properties.RetryInitialBackoff; @@ -171,7 +132,9 @@ public Service(Properties properties, SessionService.SubmitTasksWithDependencies(groupBlockRequest.Select(x => new Tuple>(x.ResultId, x.Payload! .Serialize(), - null)), + Array + .Empty< + string>())), 1, groupBlockRequest.First() .TaskOptions); @@ -196,7 +159,7 @@ public Service(Properties properties, if (ids.Count > taskIdsResultIds.Count) { - Logger?.LogWarning("Fail to submit all tasks at once, retry with missing tasks"); + Logger.LogWarning("Fail to submit all tasks at once, retry with missing tasks"); throw new Exception("Fail to submit all tasks at once. Retrying..."); } @@ -207,9 +170,9 @@ public Service(Properties properties, { if (retry >= maxRetries - 1) { - Logger?.LogError(e, - "Fail to retry {count} times of submission. Stop trying to submit", - maxRetries); + Logger.LogError(e, + "Fail to retry {count} times of submission. Stop trying to submit", + maxRetries); throw; } @@ -228,15 +191,15 @@ public Service(Properties properties, foreach (var blockRequest in groupBlockRequest) { - blockRequest.Lock?.Release(); + blockRequest.Lock.Release(); } } } catch (Exception e) { - Logger?.LogError(e, - "Fail to submit buffer with {count} tasks inside", - blockRequestList?.Count); + Logger.LogError(e, + "Fail to submit buffer with {count} tasks inside", + blockRequestList.Count); requestTaskMap_.BufferFailures(blockRequestList.Select(block => block.SubmitId), e); @@ -244,113 +207,102 @@ public Service(Properties properties, }); } - private CancellationTokenSource CancellationQueueTaskSource { get; } + /// + /// Property Get the SessionId + /// + [PublicAPI] + public SessionService SessionService { get; } private BatchUntilInactiveBlock BufferSubmit { get; } - [CanBeNull] private ILogger Logger { get; } - private ProtoSerializer ProtoSerializer { get; } - - private SessionServiceFactory SessionServiceFactory { get; set; } + private SessionServiceFactory SessionServiceFactory { get; } private CancellationTokenSource CancellationResultTaskSource { get; } /// /// The handler to send the response /// - public Task HandlerResponse { get; set; } + private Task HandlerResponse { get; } /// /// The sessionId /// public string SessionId - => SessionService?.SessionId.Id; + => SessionService.SessionId.Id; - /// - /// The method submit will execute task asynchronously on the server and will serialize object[] for Service method - /// MethodName(Object[] arguments) - /// - /// The name of the method inside the service - /// A list of object that can be passed in parameters of the function - /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error - /// Return the taskId string - public string Submit(string methodName, - object[] arguments, - IServiceInvocationHandler handler) - { - ArmonikPayload payload = new() - { - MethodName = methodName, - ClientPayload = ProtoSerializer.SerializeMessageObjectArray(arguments), - }; - var taskId = SessionService.SubmitTask(payload.Serialize()); - ResultHandlerDictionary[taskId] = handler; - return taskId; - } - /// - /// The method submit list of task with Enumerable list of arguments that will be serialized to each call of byte[] - /// MethodName(byte[] argument) - /// - /// The name of the method inside the service - /// A list of parameters that can be passed in parameters of the each call of function - /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error - /// Return the list of created taskIds + /// public IEnumerable Submit(string methodName, IEnumerable arguments, - IServiceInvocationHandler handler) - { - var armonikPayloads = arguments.Select(args => new ArmonikPayload - { - MethodName = methodName, - ClientPayload = ProtoSerializer.SerializeMessageObjectArray(args), - SerializedArguments = false, - }); + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null) + => Submit(methodName, + arguments.Select(ProtoSerializer.Serialize), + handler, + maxRetries, + null, + false); - var taskIds = SessionService.SubmitTasks(armonikPayloads.Select(p => p.Serialize())); - var submitted = taskIds as string[] ?? taskIds.ToArray(); - foreach (var taskid in submitted) - { - ResultHandlerDictionary[taskid] = handler; - } - return submitted; - } + /// + public IEnumerable Submit(string methodName, + IEnumerable arguments, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null) + => Submit(methodName, + arguments, + handler, + maxRetries, + taskOptions, + true); + + /// + public async Task SubmitAsync(string methodName, + object[] argument, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null, + CancellationToken token = default) + => await SubmitAsync(methodName, + ProtoSerializer.Serialize(argument), + handler, + maxRetries, + taskOptions, + false, + token) + .ConfigureAwait(false); - /// - /// The method submit with One serialized argument that will be already serialized for byte[] MethodName(byte[] - /// argument). - /// - /// The name of the method inside the service - /// One serialized argument that will already serialize for MethodName. - /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error - /// The number of retry before fail to submit task. Default = 5 retries - /// - /// TaskOptions argument to override default taskOptions in Session. - /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker - /// - /// Returns the taskId string - public string Submit(string methodName, - byte[] argument, - IServiceInvocationHandler handler, - int maxRetries = 5, - TaskOptions taskOptions = null) + + /// + public async Task SubmitAsync(string methodName, + byte[] argument, + IServiceInvocationHandler handler, + int maxRetries = 5, + TaskOptions? taskOptions = null, + CancellationToken token = default) + => await SubmitAsync(methodName, + ProtoSerializer.Serialize(argument), + handler, + maxRetries, + taskOptions, + true, + token) + .ConfigureAwait(false); + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public override void Dispose() { - ArmonikPayload payload = new() - { - MethodName = methodName, - ClientPayload = argument, - SerializedArguments = true, - }; - - var taskId = SessionService.SubmitTask(payload.Serialize(), - maxRetries: maxRetries, - taskOptions: taskOptions); - ResultHandlerDictionary[taskId] = handler; - return taskId; + CancellationResultTaskSource.Cancel(); + HandlerResponse.Wait(); + HandlerResponse.Dispose(); + semaphoreSlim_.Dispose(); + + GC.SuppressFinalize(this); } /// @@ -365,105 +317,61 @@ public string Submit(string methodName, /// TaskOptions argument to override default taskOptions in Session. /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// + /// defines whether the arguments should be passed as serialized to the compute function /// Return the taskId string - public IEnumerable Submit(string methodName, - IEnumerable arguments, - IServiceInvocationHandler handler, - int maxRetries = 5, - TaskOptions taskOptions = null) + private IEnumerable Submit(string methodName, + IEnumerable arguments, + IServiceInvocationHandler handler, + int maxRetries, + TaskOptions? taskOptions, + bool serializedArguments) { - var armonikPayloads = arguments.Select(args => new ArmonikPayload - { - MethodName = methodName, - ClientPayload = args, - SerializedArguments = true, - }); + var armonikPayloads = arguments.Select(args => new ArmonikPayload(methodName, + args, + serializedArguments)); var taskIds = SessionService.SubmitTasks(armonikPayloads.Select(p => p.Serialize()), maxRetries, taskOptions); var submitted = taskIds as string[] ?? taskIds.ToArray(); - foreach (var taskid in submitted) + foreach (var taskId in submitted) { - ResultHandlerDictionary[taskid] = handler; + ResultHandlerDictionary[taskId] = handler; } return submitted; } - - /// - /// The method submitAsync will serialize argument in object[] MethodName(object[] argument). - /// - /// The name of the method inside the service - /// One serialized argument that will already serialize for MethodName. - /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error - /// The number of retry before fail to submit task. Default = 5 retries - /// - /// TaskOptions argument to override default taskOptions in Session. - /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker - /// - /// The cancellation token - /// Returns the taskId string - public async Task SubmitAsync(string methodName, - object[] argument, - IServiceInvocationHandler handler, - int maxRetries = 5, - TaskOptions taskOptions = null, - CancellationToken token = default) - { - await semaphoreSlim_.WaitAsync(token); - - var blockRequest = new BlockRequest - { - SubmitId = Guid.NewGuid(), - Payload = new ArmonikPayload - { - MethodName = methodName, - ClientPayload = ProtoSerializer.SerializeMessageObjectArray(argument), - SerializedArguments = false, - }, - Handler = handler, - MaxRetries = maxRetries, - TaskOptions = taskOptions ?? SessionService.TaskOptions, - Lock = semaphoreSlim_, - }; - - return await SubmitAsync(blockRequest, - token); - } - /// /// The method submit with one serialized argument that will send as byte[] MethodName(byte[] argument). /// /// The name of the method inside the service /// One serialized argument that will already serialize for MethodName. /// The handler callBack implemented as IServiceInvocationHandler to get response or result or error - /// The cancellation token to set to cancel the async task /// The number of retry before fail to submit task. Default = 5 retries /// /// TaskOptions argument to override default taskOptions in Session. /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// + /// defines whether the arguments should be passed as serialized to the compute function + /// The cancellation token to set to cancel the async task /// Return the taskId string - public async Task SubmitAsync(string methodName, - byte[] argument, - IServiceInvocationHandler handler, - int maxRetries = 5, - TaskOptions taskOptions = null, - CancellationToken token = default) + private async Task SubmitAsync(string methodName, + byte[] argument, + IServiceInvocationHandler handler, + int maxRetries, + TaskOptions? taskOptions, + bool serializedArguments, + CancellationToken token) { await semaphoreSlim_.WaitAsync(token); return await SubmitAsync(new BlockRequest { SubmitId = Guid.NewGuid(), - Payload = new ArmonikPayload - { - MethodName = methodName, - ClientPayload = argument, - SerializedArguments = true, - }, + Payload = new ArmonikPayload(methodName, + argument, + serializedArguments), Handler = handler, MaxRetries = maxRetries, TaskOptions = taskOptions ?? SessionService.TaskOptions, @@ -492,28 +400,24 @@ await BufferSubmit.SendAsync(blockRequest, /// the array of object to pass as arguments for the method /// Returns an object as result of the method call /// - [CanBeNull] - public ServiceResult LocalExecute(object service, + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedMember.Global +#pragma warning disable CA1822 + public ServiceResult LocalExecute(object service, +#pragma warning restore CA1822 string methodName, object[] arguments) { var methodInfo = service.GetType() - .GetMethod(methodName); + .GetMethod(methodName) ?? throw new InvalidOperationException($"MethodName [{methodName}] was not found"); - if (methodInfo == null) - { - throw new InvalidOperationException($"MethodName [{methodName}] was not found"); - } var result = methodInfo.Invoke(service, - arguments); - - return new ServiceResult - { - TaskId = Guid.NewGuid() - .ToString(), - Result = result, - }; + arguments)!; + + return new ServiceResult(Guid.NewGuid() + .ToString(), + result); } /// @@ -528,29 +432,41 @@ public ServiceResult LocalExecute(object service, /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// /// Returns a tuple with the taskId string and an object as result of the method call - public ServiceResult Execute(string methodName, - object[] arguments, - int maxRetries = 5, - TaskOptions taskOptions = null) - { - ArmonikPayload unifiedPayload = new() - { - MethodName = methodName, - ClientPayload = ProtoSerializer.SerializeMessageObjectArray(arguments), - }; - - var taskId = SessionService.SubmitTask(unifiedPayload.Serialize(), - maxRetries: maxRetries, - taskOptions: taskOptions); + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedMember.Global + public ServiceResult Execute(string methodName, + object[] arguments, + int maxRetries = 5, + TaskOptions? taskOptions = null) + => Execute(methodName, + ProtoSerializer.Serialize(arguments), + maxRetries, + taskOptions, + false); - var result = ProtoSerializer.DeSerializeMessageObjectArray(SessionService.GetResult(taskId)); - - return new ServiceResult - { - TaskId = taskId, - Result = result?[0], - }; - } + /// + /// This method is used to execute task and waiting after the result. + /// the method will return the result of the execution until the grid returns the task result + /// + /// The string name of the method + /// the array of byte to pass as argument for the methodName(byte[] dataArg) + /// The number of retry before fail to submit task. Default = 5 retries + /// + /// TaskOptions argument to override default taskOptions in Session. + /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker + /// + /// Returns a tuple with the taskId string and an object as result of the method call + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedMember.Global + public ServiceResult Execute(string methodName, + byte[] dataArg, + int maxRetries = 5, + TaskOptions? taskOptions = null) + => Execute(methodName, + dataArg, + maxRetries, + taskOptions, + true); /// /// This method is used to execute task and waiting after the result. @@ -563,21 +479,20 @@ public ServiceResult Execute(string methodName, /// TaskOptions argument to override default taskOptions in Session. /// If non null it will override the default taskOptions in SessionService for client or given by taskHandler for worker /// + /// defines whether the arguments should be passed as serialized to the compute function /// Returns a tuple with the taskId string and an object as result of the method call - public ServiceResult Execute(string methodName, - byte[] dataArg, - int maxRetries = 5, - TaskOptions taskOptions = null) + private ServiceResult Execute(string methodName, + byte[] dataArg, + int maxRetries, + TaskOptions? taskOptions, + bool serializedArguments) { - ArmonikPayload unifiedPayload = new() - { - MethodName = methodName, - ClientPayload = dataArg, - SerializedArguments = true, - }; + ArmonikPayload unifiedPayload = new(methodName, + dataArg, + serializedArguments); - var taskId = "not-TaskId"; - object[] result; + var taskId = "not-TaskId"; + object?[] result; try { @@ -585,14 +500,15 @@ public ServiceResult Execute(string methodName, maxRetries: maxRetries, taskOptions: taskOptions); - result = ProtoSerializer.DeSerializeMessageObjectArray(SessionService.GetResult(taskId)); + result = ProtoSerializer.Deserialize(SessionService.GetResult(taskId))!; } catch (Exception e) { var status = SessionService.GetTaskStatus(taskId); - var details = ""; + var details = string.Empty; + // ReSharper disable once InvertIf if (status != TaskStatus.Completed) { var output = SessionService.GetTaskOutputInfo(taskId); @@ -602,21 +518,16 @@ public ServiceResult Execute(string methodName, } throw new ServiceInvocationException(e is AggregateException - ? e.InnerException + ? e.InnerException ?? e : e, - StatusCodesLookUp.Keys.Contains(status) - ? StatusCodesLookUp[status] - : ArmonikStatusCode.Unknown) + status.ToArmonikStatusCode()) { OutputDetails = details, }; } - return new ServiceResult - { - TaskId = taskId, - Result = result?[0], - }; + return new ServiceResult(taskId, + result[0]); } /// @@ -631,7 +542,7 @@ private void ProxyTryGetResults(IEnumerable taskIds, Action errorHandler, int chunkResultSize = 200) { - var missing = taskIds.ToHashSet(); + var missing = new HashSet(taskIds); var holdPrev = missing.Count; var waitInSeconds = new List { @@ -655,8 +566,8 @@ private void ProxyTryGetResults(IEnumerable taskIds, { try { - Logger?.LogTrace("Response handler for {taskId}", - resultStatusData.TaskId); + Logger.LogTrace("Response handler for {taskId}", + resultStatusData.TaskId); responseHandler(resultStatusData.TaskId, Retry.WhileException(5, 2000, @@ -664,9 +575,9 @@ private void ProxyTryGetResults(IEnumerable taskIds, { if (retry > 1) { - Logger?.LogWarning("Try {try} for {funcName}", - retry, - nameof(SessionService.TryGetResultAsync)); + Logger.LogWarning("Try {try} for {funcName}", + retry, + nameof(SessionService.TryGetResultAsync)); } return SessionService.TryGetResultAsync(new ResultRequest @@ -680,13 +591,13 @@ private void ProxyTryGetResults(IEnumerable taskIds, true, Logger, typeof(IOException), - typeof(RpcException))); + typeof(RpcException))!); } catch (Exception e) { - Logger?.LogWarning(e, - "Response handler for {taskId} threw an error", - resultStatusData.TaskId); + Logger.LogWarning(e, + "Response handler for {taskId} threw an error", + resultStatusData.TaskId); try { errorHandler(resultStatusData.TaskId, @@ -695,9 +606,9 @@ private void ProxyTryGetResults(IEnumerable taskIds, } catch (Exception e2) { - Logger?.LogError(e2, - "An error occured while handling another error: {details}", - e); + Logger.LogError(e2, + "An error occurred while handling another error: {details}", + e); } } } @@ -724,10 +635,10 @@ private void ProxyTryGetResults(IEnumerable taskIds, break; } - Logger?.LogDebug("Error handler for {taskId}, {taskStatus}: {details}", - resultStatusData.TaskId, - taskStatus, - details); + Logger.LogDebug("Error handler for {taskId}, {taskStatus}: {details}", + resultStatusData.TaskId, + taskStatus, + details); try { errorHandler(resultStatusData.TaskId, @@ -736,10 +647,10 @@ private void ProxyTryGetResults(IEnumerable taskIds, } catch (Exception e) { - Logger?.LogError(e, - "An error occured while handling a Task error {status}: {details}", - taskStatus, - details); + Logger.LogError(e, + "An error occurred while handling a Task error {status}: {details}", + taskStatus, + details); } } @@ -755,10 +666,10 @@ private void ProxyTryGetResults(IEnumerable taskIds, } catch (Exception e) { - Logger?.LogError(e, - "An error occured while handling a Task error {status}: {details}", - TaskStatus.Unspecified, - "Task is missing"); + Logger.LogError(e, + "An error occurred while handling a Task error {status}: {details}", + TaskStatus.Unspecified, + "Task is missing"); } } @@ -770,8 +681,8 @@ private void ProxyTryGetResults(IEnumerable taskIds, ? waitInSeconds.Count - 1 : idx + 1; - Logger?.LogDebug("No Results are ready. Wait for {timeWait} seconds before new retry", - waitInSeconds[idx] / 1000); + Logger.LogDebug("No Results are ready. Wait for {timeWait} seconds before new retry", + waitInSeconds[idx] / 1000); } else { @@ -799,9 +710,9 @@ private void ResultTask() { try { - var result = ProtoSerializer.DeSerializeMessageObjectArray(byteResult); + var result = ProtoSerializer.Deserialize(byteResult); ResultHandlerDictionary[taskId] - .HandleResponse(result?[0], + .HandleResponse(result![0], taskId); } catch (Exception e) @@ -819,7 +730,7 @@ private void ResultTask() } else if (ae is not null) { - ex = new ServiceInvocationException(ae.InnerException, + ex = new ServiceInvocationException(ae.InnerException ?? ae, statusCode); } else @@ -844,9 +755,7 @@ private void ResultTask() { try { - var statusCode = StatusCodesLookUp.Keys.Contains(taskStatus) - ? StatusCodesLookUp[taskStatus] - : ArmonikStatusCode.Unknown; + var statusCode = taskStatus.ToArmonikStatusCode(); ResultHandlerDictionary[taskId] .HandleError(new ServiceInvocationException(ex, @@ -867,16 +776,16 @@ private void ResultTask() } catch (Exception e) { - Logger?.LogError("An error occured while fetching results: {e}", - e); + Logger.LogError("An error occurred while fetching results: {e}", + e); } } if (!ResultHandlerDictionary.IsEmpty) { - Logger?.LogWarning("Results not processed : [{resultsNotProcessed}]", - string.Join(", ", - ResultHandlerDictionary.Keys)); + Logger.LogWarning("Results not processed : [{resultsNotProcessed}]", + string.Join(", ", + ResultHandlerDictionary.Keys)); } } @@ -885,54 +794,40 @@ private void ResultTask() /// Get a new channel to communicate with the control plane /// /// gRPC channel + // TODO: Refactor test to remove this + // ReSharper disable once UnusedMember.Global public ChannelBase GetChannel() => SessionService.ChannelPool.GetChannel(); - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - public override void Dispose() - { - CancellationResultTaskSource.Cancel(); - HandlerResponse?.Wait(); - HandlerResponse?.Dispose(); - - SessionService = null; - SessionServiceFactory = null; - semaphoreSlim_.Dispose(); - } - /// - /// The method to destroy the service and close the session - /// - public void Destroy() - => Dispose(); - - /// - /// Check if this service has been destroyed before that call + /// Class to return TaskId and the result /// - /// Returns true if the service was destroyed previously - public bool IsDestroyed() + public class ServiceResult { - if (SessionService == null || SessionServiceFactory == null) + /// + /// Constructor + /// + /// + /// + public ServiceResult(string taskId, + object? result) { - return true; + TaskId = taskId; + Result = result; } - return false; - } - - /// - /// Class to return TaskId and the result - /// - public class ServiceResult - { /// /// The getter to return the taskId /// - public string TaskId { get; set; } + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedAutoPropertyAccessor.Global + public string TaskId { get; } /// /// The getter to return the result in object type format /// - public object Result { get; set; } + // TODO: mark with [PublicApi] ? + // ReSharper disable once UnusedAutoPropertyAccessor.Global + public object? Result { get; } } } diff --git a/Common/src/Common/AppsOptions.cs b/Common/src/Common/AppsOptions.cs index 09f3f9a7..5d12b947 100644 --- a/Common/src/Common/AppsOptions.cs +++ b/Common/src/Common/AppsOptions.cs @@ -20,7 +20,9 @@ namespace ArmoniK.DevelopmentKit.Common; [MarkDownDoc] public static class AppsOptions { - public const string GridVolumesKey = "gridVolumes"; - public const string GridAppVolumesKey = "target_app_path"; - public const string GridDataVolumesKey = "target_data_path"; + public const string GridVolumesKey = "gridVolumes"; + public const string GridAppVolumesKey = "target_app_path"; + public const string GridDataVolumesKey = "target_data_path"; + public const string GridAssemblyPathKey = "ServiceAssemblyPath"; + public const string GridZipVolumePath = "target_zip_path"; } diff --git a/Common/src/Common/ArmoniK.DevelopmentKit.Common.csproj b/Common/src/Common/ArmoniK.DevelopmentKit.Common.csproj index c844fe15..e731ce79 100644 --- a/Common/src/Common/ArmoniK.DevelopmentKit.Common.csproj +++ b/Common/src/Common/ArmoniK.DevelopmentKit.Common.csproj @@ -1,7 +1,7 @@ - net472;net48;net5.0;net6.0 + netstandard2.0 Library True true @@ -9,10 +9,8 @@ - - diff --git a/Common/src/Common/ArmoniKPayload.cs b/Common/src/Common/ArmoniKPayload.cs index d447ce4e..1f105915 100644 --- a/Common/src/Common/ArmoniKPayload.cs +++ b/Common/src/Common/ArmoniKPayload.cs @@ -14,43 +14,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; - using ProtoBuf; namespace ArmoniK.DevelopmentKit.Common; -[ProtoContract] -public class ArmonikPayload +/// +/// A class used to define the computation required from the worker +/// +/// The name of the method to execute +/// The arguments for the method +/// Defines whether the payload should be transmitted as is to the worker method. +[ProtoContract(SkipConstructor = true)] +public record ArmonikPayload([property: ProtoMember(1)] + string MethodName, + [property: ProtoMember(2)] + byte[] ClientPayload, + [property: ProtoMember(3)] + bool SerializedArguments) { - [ProtoMember(1)] - public string MethodName { get; set; } - - [ProtoMember(2)] - public byte[] ClientPayload { get; set; } - - [ProtoMember(3)] - public bool SerializedArguments { get; set; } - public byte[] Serialize() - { - if (ClientPayload is null) - { - throw new ArgumentNullException(nameof(ClientPayload)); - } - - var result = ProtoSerializer.SerializeMessageObject(this); - - return result; - } + => ProtoSerializer.Serialize(this); public static ArmonikPayload Deserialize(byte[] payload) - { - if (payload == null || payload.Length == 0) - { - return new ArmonikPayload(); - } - - return ProtoSerializer.Deserialize(payload); - } + => ProtoSerializer.Deserialize(payload)!; } diff --git a/Common/src/Common/Exceptions/ClientResultsException.cs b/Common/src/Common/Exceptions/ClientResultsException.cs index 2ff7ff30..fdfebb5e 100644 --- a/Common/src/Common/Exceptions/ClientResultsException.cs +++ b/Common/src/Common/Exceptions/ClientResultsException.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2023.All rights reserved. // @@ -46,17 +46,21 @@ public ClientResultsException(string message, /// /// The default constructor to refer the list of task in error /// - /// The string message in exception - /// - public ClientResultsException(string message, - IEnumerable taskIds) - : base(message) + /// the message in exception + /// Exception that caused this one to be raised + /// The list of taskId + public ClientResultsException(string message, + Exception innerException, + params string[] taskIds) + : base(message, + innerException) => TaskIds = taskIds; + /// /// The list of taskId in error /// - public IEnumerable TaskIds { get; set; } + public string[] TaskIds { get; } private static string BuildMessage(IEnumerable taskIds) { diff --git a/Common/src/Common/Portability/IsExternalInit.cs b/Common/src/Common/Portability/IsExternalInit.cs new file mode 100644 index 00000000..a0c89d77 --- /dev/null +++ b/Common/src/Common/Portability/IsExternalInit.cs @@ -0,0 +1,25 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2023. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#if NETFRAMEWORK || NETSTANDARD +// This type is required to use initializers when compiling to framework +namespace System.Runtime.CompilerServices; + +internal static class IsExternalInit +{ +} +#endif diff --git a/Common/src/Common/ProtoSerializer.cs b/Common/src/Common/ProtoSerializer.cs index 70555fb4..61339eff 100644 --- a/Common/src/Common/ProtoSerializer.cs +++ b/Common/src/Common/ProtoSerializer.cs @@ -15,73 +15,47 @@ // limitations under the License. using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.Serialization; using ArmoniK.DevelopmentKit.Common.Exceptions; using ProtoBuf; -#pragma warning disable CS1591 - - namespace ArmoniK.DevelopmentKit.Common; -public class ProtoSerializer +// TODO: should it be marked for [PublicApi] ? +public static class ProtoSerializer { // *** you need some mechanism to map types to fields - private static readonly IDictionary typeLookup = new List - { - typeof(ProtoNative), - typeof(int), - typeof(uint), - typeof(long), - typeof(ulong), - typeof(double), - typeof(float), - typeof(short), - typeof(byte), - typeof(string), - typeof(int[]), - typeof(uint[]), - typeof(long[]), - typeof(ulong[]), - typeof(double[]), - typeof(float[]), - typeof(short[]), - typeof(byte[]), - typeof(string[]), - typeof(Nullable), - typeof(ProtoArray), - typeof(IEnumerable), - typeof(IDictionary), - typeof(Array), - typeof(ArmonikPayload), - }.Select((t, - idx) => new - { - idx, - t, - }) - .ToDictionary(x => x.idx, - x => x.t); - - public byte[] SerializeMessageObjectArray(object[] values) - { - using var ms = new MemoryStream(); - foreach (var obj in values) - { - WriteNext(ms, - obj); - } - - var data = ms.ToArray(); - return data; - } - - public static byte[] SerializeMessageObject(object value) + private static readonly List TypeLookup = new() + { + typeof(int), + typeof(uint), + typeof(long), + typeof(ulong), + typeof(double), + typeof(float), + typeof(short), + typeof(byte), + typeof(string), + typeof(int[]), + typeof(uint[]), + typeof(long[]), + typeof(ulong[]), + typeof(double[]), + typeof(float[]), + typeof(short[]), + typeof(byte[]), + typeof(string[]), + typeof(Nullable), + typeof(ProtoArray), + typeof(ArmonikPayload), + }; + + public static byte[] Serialize(object? value) { using var ms = new MemoryStream(); @@ -93,45 +67,27 @@ public static byte[] SerializeMessageObject(object value) return data; } - public static object[] DeSerializeMessageObjectArray(byte[] data) + // TODO: is it [PublicApi]? + // ReSharper disable once UnusedMember.Global + public static void RegisterClass(Type type) { - var result = new List(); - - using var ms = new MemoryStream(data); - while (ReadNext(ms, - out var obj)) + if (TypeLookup.Contains(type)) { - result.Add(obj); + throw new ArgumentException("Type already registered", + nameof(type)); } - return result.Count == 0 - ? null - : result.ToArray(); + TypeLookup.Add(type); } - public static object DeSerializeMessageObject(byte[] data) - { - using var ms = new MemoryStream(data); - - ReadNext(ms, - out var obj); - return obj; - } - - public static void RegisterClass(Type classType) - { - var max = typeLookup.Keys.Max(); - typeLookup[max + 1] = classType; - } - - private static void WriteNext(Stream stream, - object obj) + private static void WriteNext(Stream stream, + object? obj) { obj ??= new Nullable(); var type = obj.GetType(); - if (type.IsArray && typeLookup.All(pair => pair.Value.Name != type.Name)) + if (type.IsArray && TypeLookup.All(t => t.Name != type.Name)) { WriteNext(stream, new ProtoArray @@ -157,65 +113,69 @@ private static void SerializeSingle(Stream stream, object obj, Type type) { - var field = typeLookup.Single(pair => pair.Value == type) - .Key; - + // "+1" to have 1-based indexing instead of 0-based indexing. Required by protocol buffer. + var field = TypeLookup.IndexOf(type) + 1; Serializer.NonGeneric.SerializeWithLengthPrefix(stream, obj, PrefixStyle.Base128, field); } - private static bool ReadNext(Stream stream, - out object obj) + private static bool ReadNext(Stream stream, + out object? obj) { if (!Serializer.NonGeneric.TryDeserializeWithLengthPrefix(stream, PrefixStyle.Base128, - field => - { - return typeLookup[field]; - }, + // "-1" to have 1-based indexing instead of 0-based indexing. Required by protocol buffer. + field => TypeLookup[field - 1], out obj)) { return false; } - if (obj is Nullable) + switch (obj) { - obj = null; - } + case Nullable: + obj = null; + break; - if (obj is ProtoArray) - { - var finalObj = new List(); - var arrInfo = (ProtoArray)obj; - if (arrInfo.NbElement < 0) + case ProtoArray arrInfo: { - throw new WorkerApiException($"ProtoArray failure number of element [{arrInfo.NbElement}] < 0 "); - } + var finalObj = new List(); + if (arrInfo.NbElement < 0) + { + throw new WorkerApiException($"ProtoArray failure number of element [{arrInfo.NbElement}] < 0 "); + } - for (var i = 0; i < arrInfo.NbElement; i++) - { - if (!ReadNext(stream, - out var subObj)) + for (var i = 0; i < arrInfo.NbElement; i++) { - throw new WorkerApiException($"Fail to iterate over ProtoArray with Element {arrInfo.NbElement} at index [{i}]"); + if (!ReadNext(stream, + out var subObj)) + { + throw new WorkerApiException($"Fail to iterate over ProtoArray with Element {arrInfo.NbElement} at index [{i}]"); + } + + finalObj.Add(subObj); } - finalObj.Add(subObj); + obj = finalObj.ToArray(); + break; } - - obj = finalObj.ToArray(); } return true; } - public static T Deserialize(byte[] dataPayloadInBytes) + public static T? Deserialize(byte[] dataPayloadInBytes) { - var obj = DeSerializeMessageObject(dataPayloadInBytes); + using var ms = new MemoryStream(dataPayloadInBytes); + if (!ReadNext(ms, + out var obj)) + { + throw new SerializationException("Error while deserializing object."); + } - return (T)obj; + return (T?)obj; } [ProtoContract] @@ -224,16 +184,9 @@ public class Nullable } [ProtoContract] - public class ProtoArray + private class ProtoArray { [ProtoMember(1)] public int NbElement; } - - [ProtoContract] - public class ProtoNative - { - [ProtoMember(1)] - public T Element; - } } diff --git a/Common/src/Common/Utils/FileSpinLock.cs b/Common/src/Common/Utils/FileSpinLock.cs new file mode 100644 index 00000000..488ff4ca --- /dev/null +++ b/Common/src/Common/Utils/FileSpinLock.cs @@ -0,0 +1,109 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2023. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Text; +using System.Threading; + +using JetBrains.Annotations; + +namespace ArmoniK.DevelopmentKit.Common.Utils; + +/// +/// Spin lock on a file +/// +[PublicAPI] +public sealed class FileSpinLock : IDisposable +{ + private static readonly byte[] LockBytes = Encoding.ASCII.GetBytes("locked"); + + [CanBeNull] + private readonly FileStream fileStream_; + + /// + /// Creates a spinlock on the file. Sets to true if it successfully locked the file, false + /// otherwise + /// + /// File to lock + /// Delete the lockfile when this object is disposed + /// Maximum time to wait for the lock to be acquired + /// Interval between lock tries + public FileSpinLock(string lockFile, + bool deleteOnUnlock = true, + int timeoutMs = 30000, + int spinIntervalMs = 250) + { + HasLock = false; + var currentSpinTime = 0; + spinIntervalMs = Math.Min(spinIntervalMs, + timeoutMs); + do + { + try + { + fileStream_ ??= new FileStream(lockFile, + FileMode.OpenOrCreate, + FileAccess.ReadWrite, + FileShare.None, + 1, + FileOptions.WriteThrough | (deleteOnUnlock + ? FileOptions.DeleteOnClose + : FileOptions.None)); + if (fileStream_.Seek(0, + SeekOrigin.End) == 0) + { + fileStream_.Write(LockBytes, + 0, + LockBytes.Length); + fileStream_.Flush(); + } + + fileStream_.Lock(0, + LockBytes.Length); + + HasLock = true; + } + catch (IOException) + { + Thread.Sleep(spinIntervalMs); + currentSpinTime += spinIntervalMs; + } + catch (UnauthorizedAccessException) + { + Thread.Sleep(spinIntervalMs); + currentSpinTime += spinIntervalMs; + } + } while (!HasLock && currentSpinTime < timeoutMs + spinIntervalMs); + } + + /// + /// True if the file is locked by the current class, false otherwise + /// + public bool HasLock { get; } + + /// + public void Dispose() + { + if (HasLock) + { + fileStream_?.Unlock(0, + LockBytes.Length); + } + + fileStream_?.Dispose(); + } +} diff --git a/Directory.Build.props b/Directory.Build.props index 0f2d527a..07e56ff0 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -62,6 +62,16 @@ true true true + true + + + true + enable + 10 + @@ -71,14 +81,16 @@ - Embedded - true DEBUG;TRACE + + true + + diff --git a/Documentation/.gitignore b/Documentation/.gitignore new file mode 100644 index 00000000..c3f46aab --- /dev/null +++ b/Documentation/.gitignore @@ -0,0 +1,10 @@ +############### +# folder # +############### +/**/DROP/ +/**/TEMP/ +/**/packages/ +/**/bin/ +/**/obj/ +_site + diff --git a/Documentation/api/.gitignore b/Documentation/api/.gitignore new file mode 100644 index 00000000..e8079a3b --- /dev/null +++ b/Documentation/api/.gitignore @@ -0,0 +1,5 @@ +############### +# temp file # +############### +*.yml +.manifest diff --git a/Documentation/api/index.md b/Documentation/api/index.md new file mode 100644 index 00000000..f83c9b6f --- /dev/null +++ b/Documentation/api/index.md @@ -0,0 +1,2 @@ +# API documentation + diff --git a/Documentation/articles/intro.md b/Documentation/articles/intro.md new file mode 100644 index 00000000..c0478ced --- /dev/null +++ b/Documentation/articles/intro.md @@ -0,0 +1 @@ +# Add your introductions here! diff --git a/Documentation/articles/toc.yml b/Documentation/articles/toc.yml new file mode 100644 index 00000000..c239cb2a --- /dev/null +++ b/Documentation/articles/toc.yml @@ -0,0 +1,2 @@ +- name: Buffering submission + href: buffersubmit.md diff --git a/Documentation/docfx.json b/Documentation/docfx.json new file mode 100644 index 00000000..308d293b --- /dev/null +++ b/Documentation/docfx.json @@ -0,0 +1,75 @@ +{ + "metadata": [ + { + "src": [ + { + "src": "../", + "files": [ + "**.csproj" + ] + } + ], + "dest": "api", + "includePrivateMembers": false, + "disableGitFeatures": false, + "disableDefaultFilter": false, + "noRestore": false, + "namespaceLayout": "flattened", + "memberLayout": "samePage", + "allowCompilationErrors": false + } + ], + "build": { + "content": [ + { + "files": [ + "api/**.yml", + "api/index.md" + ] + }, + { + "files": [ + "articles/**.md", + "articles/**/toc.yml", + "toc.yml", + "*.md" + ] + } + ], + "resource": [ + { + "files": [ + "images/**" + ] + } + ], + "overwrite": [ + { + "files": [ + "apidoc/**.md" + ], + "exclude": [ + "obj/**", + "_site/**" + ] + } + ], + "output": "_site", + "globalMetadata": { + "_appFaviconPath": "images/armonik_favicon.ico", + "_appLogoPath": "images/armonik_logo.svg", + "_appFooter": "Aneo", + "_copyrightFooter": "© Aneo. All rights reserved.", + "_enableSearch": true + }, + "globalMetadataFiles": [], + "fileMetadataFiles": [], + "template": [ + "default", + "modern" + ], + "postProcessors": [], + "keepFileLink": false, + "disableGitFeatures": false + } +} diff --git a/Documentation/images/armonik_favicon.ico b/Documentation/images/armonik_favicon.ico new file mode 100644 index 00000000..b9b2fdac Binary files /dev/null and b/Documentation/images/armonik_favicon.ico differ diff --git a/Documentation/images/armonik_logo.svg b/Documentation/images/armonik_logo.svg new file mode 100644 index 00000000..1eebd89a --- /dev/null +++ b/Documentation/images/armonik_logo.svg @@ -0,0 +1,203 @@ + + + ArmoniK logo + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + ArmoniK logo + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Documentation/index.md b/Documentation/index.md new file mode 100644 index 00000000..c4634558 --- /dev/null +++ b/Documentation/index.md @@ -0,0 +1,7 @@ +# ArmoniK.Extensions.Csharp API documentation + +Autogenerated documentation of the Extensions.Csharp for ArmoniK + +## Articles + +- [Buffering submission](articles/buffersubmit.md) diff --git a/Documentation/toc.yml b/Documentation/toc.yml new file mode 100644 index 00000000..59f80104 --- /dev/null +++ b/Documentation/toc.yml @@ -0,0 +1,5 @@ +- name: Articles + href: articles/ +- name: Api Documentation + href: api/ + homepage: api/index.md diff --git a/Tests/ArmoniK.DevelopmentKit.Common.Tests/ArmoniK.DevelopmentKit.Common.Tests.csproj b/Tests/ArmoniK.DevelopmentKit.Common.Tests/ArmoniK.DevelopmentKit.Common.Tests.csproj new file mode 100644 index 00000000..28c922d9 --- /dev/null +++ b/Tests/ArmoniK.DevelopmentKit.Common.Tests/ArmoniK.DevelopmentKit.Common.Tests.csproj @@ -0,0 +1,26 @@ + + + + net472;net6.0 + enable + + false + true + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/Tests/ArmoniK.DevelopmentKit.Common.Tests/Directory.Build.props b/Tests/ArmoniK.DevelopmentKit.Common.Tests/Directory.Build.props new file mode 100644 index 00000000..448d517c --- /dev/null +++ b/Tests/ArmoniK.DevelopmentKit.Common.Tests/Directory.Build.props @@ -0,0 +1,11 @@ + + + + ../../ + + + + + + + diff --git a/Tests/ArmoniK.DevelopmentKit.Common.Tests/ProtoSerializerTest.cs b/Tests/ArmoniK.DevelopmentKit.Common.Tests/ProtoSerializerTest.cs new file mode 100644 index 00000000..e44ae5c1 --- /dev/null +++ b/Tests/ArmoniK.DevelopmentKit.Common.Tests/ProtoSerializerTest.cs @@ -0,0 +1,209 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-$CURRENT_YEAR$. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using NUnit.Framework; + +namespace ArmoniK.DevelopmentKit.Common.Tests; + +public class ProtoSerializerTest +{ + [Test] + [TestCase(1)] + [TestCase(1U)] + [TestCase(1L)] + [TestCase(1UL)] + [TestCase(1.0)] + [TestCase(1.0f)] + [TestCase((short)1)] + [TestCase((byte)1)] + [TestCase("test")] + public void SerializeAndDeserialize(T? message) + { + var serialized = ProtoSerializer.Serialize(message); + var result = ProtoSerializer.Deserialize(serialized); + Assert.That(result, + Is.EqualTo(message)); + } + + [Test] + [TestCase(1)] + [TestCase(1U)] + [TestCase(1L)] + [TestCase(1UL)] + [TestCase(1.0)] + [TestCase(1.0f)] + [TestCase((short)1)] + [TestCase((byte)1)] + [TestCase("test")] + public void SerializeAndDeserializeArrayTypes(T? message) + { + var serialized = ProtoSerializer.Serialize(new[] + { + message, + }); + var result = ProtoSerializer.Deserialize(serialized); + Assert.That(result, + Is.Not.Null); + Assert.That(result!.Length, + Is.EqualTo(1)); + Assert.That(result![0], + Is.EqualTo(message)); + } + + [Test] + public void SerializeAndDeserializeArray() + { + var message = Enumerable.Range(0, + 5) + .ToArray() as Array; + var serialized = ProtoSerializer.Serialize(message); + var result = ProtoSerializer.Deserialize(serialized); + Assert.That(result, + Is.Not.Null); + + var array = result.Cast() + .ToArray(); + + Assert.That(array.Count, + Is.EqualTo(5)); + Assert.Multiple(() => + { + Assert.That(array[0], + Is.EqualTo(0)); + Assert.That(array[1], + Is.EqualTo(1)); + Assert.That(array[2], + Is.EqualTo(2)); + Assert.That(array[3], + Is.EqualTo(3)); + Assert.That(array[4], + Is.EqualTo(4)); + }); + } + + + [Test] + public void SerializeAndDeserializeArmonikPayload() + { + var message = new ArmonikPayload("methodName", + new[] + { + (byte)0x01, + (byte)0x02, + }, + false); + + var serialized = ProtoSerializer.Serialize(message); + var result = ProtoSerializer.Deserialize(serialized); + Assert.That(result, + Is.Not.Null); + + Assert.Multiple(() => + { + Assert.That(result!.ClientPayload, + Is.Not.Null); + Assert.That(result!.ClientPayload[0], + Is.EqualTo((byte)0x01)); + Assert.That(result!.ClientPayload[1], + Is.EqualTo((byte)0x02)); + Assert.That(result!.MethodName, + Is.EqualTo("methodName")); + Assert.That(result!.SerializedArguments, + Is.EqualTo(false)); + }); + } + + + [Test] + public void SerializeAndDeserializeArrayofArray() + { + var message = Enumerable.Range(0, + 3) + .Select(i => Enumerable.Range(0, + 2) + .Select(i1 => $"{i},{i1}") + .ToArray()) + .Append(null) + .ToArray(); + var serialized = ProtoSerializer.Serialize(message); + var result = ProtoSerializer.Deserialize(serialized); + Assert.That(result, + Is.Not.Null); + + Assert.Multiple(() => + { + Assert.That(result![0], + Is.Not.Null); + Assert.That(result![0], + Is.TypeOf()); + Assert.That(result![1], + Is.Not.Null); + Assert.That(result![1], + Is.TypeOf()); + Assert.That(result![2], + Is.Not.Null); + Assert.That(result![2], + Is.TypeOf()); + Assert.That(result![3], + Is.Null); + Assert.That((result![0] as string[])![0], + Is.EqualTo("0,0")); + Assert.That((result![0] as string[])![1], + Is.EqualTo("0,1")); + Assert.That((result![1] as string[])![0], + Is.EqualTo("1,0")); + Assert.That((result![1] as string[])![1], + Is.EqualTo("1,1")); + Assert.That((result![2] as string[])![0], + Is.EqualTo("2,0")); + Assert.That((result![2] as string[])![1], + Is.EqualTo("2,1")); + }); + } + + [Test] + public void SerializeAndDeserializeMixedArray() + { + var list = new List(); + list.Add(5); + list.Add(new[] + { + 1.1, + 1.2, + }); + var array = list.ToArray(); + + + var serialized = ProtoSerializer.Serialize(array); + var result = ProtoSerializer.Deserialize(serialized); + + Assert.That(result, + Is.Not.Null); + Assert.That(result, + Is.Not.Empty); + Assert.That(result![0]!.GetType(), + Is.EqualTo(typeof(int))); + Assert.That(result![0], + Is.EqualTo(5)); + Assert.That(result![1]!.GetType(), + Is.EqualTo(typeof(double[]))); + Assert.That((result![1] as double[])![0], + Is.EqualTo(1.1)); + Assert.That((result![1] as double[])![1], + Is.EqualTo(1.2)); + } +} diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/ArmoniK.EndToEndTests.Client.csproj b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/ArmoniK.EndToEndTests.Client.csproj index 5bebe4d5..0bfd2cc9 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/ArmoniK.EndToEndTests.Client.csproj +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/ArmoniK.EndToEndTests.Client.csproj @@ -4,7 +4,6 @@ net472;net6.0 1.0.0-700 - 10 true true win-x64 @@ -27,7 +26,6 @@ - @@ -44,8 +42,8 @@ - + diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs index 40ec37c0..1d0b7e4a 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.DevelopmentKit.Client.Common.Status; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; using ArmoniK.EndToEndTests.Common; @@ -152,7 +153,6 @@ private async IAsyncEnumerable RetrieveAllTasksStats(ChannelBase private async Task> GetDistribution(int nRows) { var service = unifiedTestHelper_.Service as Service; - service.GetChannel(); var taskRawData = new List(); @@ -337,7 +337,7 @@ private IEnumerable> WaitForResults(string se Assert.IsNotNull(result); var taskResults = result.Select(tp => { - var armonikPayload = ProtoSerializer.DeSerializeMessageObjectArray(tp.Item2); + var armonikPayload = ProtoSerializer.Deserialize(tp.Item2); return (tp.Item1, taskDataIds.First(taskData => tp.Item1 == taskData.Id), TaskResult.Deserialize(armonikPayload[0] as byte[])); }); @@ -369,7 +369,7 @@ public void Check_That_Result_has_expected_value(int squareMatrixSize) Assert.IsNotNull(result); - var deprot = ProtoSerializer.DeSerializeMessageObjectArray(result.Item2); + var deprot = ProtoSerializer.Deserialize(result.Item2); var taskResult = TaskResult.Deserialize(deprot[0] as byte[]); unifiedTestHelper_.Log.LogInformation($"Result of Matrix formula : {taskResult.Result}"); diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/ArmoniKPayloadSerialization.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/ArmoniKPayloadSerialization.cs index e5740a7f..b736f5e2 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/ArmoniKPayloadSerialization.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/ArmoniKPayloadSerialization.cs @@ -26,12 +26,9 @@ public static class ArmoniKPayloadSerialization [Test] public static void ShouldSerialize() { - var payload = new ArmonikPayload - { - MethodName = "Test", - ClientPayload = Encoding.ASCII.GetBytes("Payload"), - SerializedArguments = true, - }; + var payload = new ArmonikPayload("Test", + Encoding.ASCII.GetBytes("Payload"), + true); var serialize = payload.Serialize(); var deserialize = ProtoSerializer.Deserialize(serialize); diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClient.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClient.cs index efbb9bc3..486167a2 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClient.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClient.cs @@ -22,8 +22,8 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Factory; -using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; using ArmoniK.EndToEndTests.Common; @@ -116,9 +116,9 @@ private static object[] ParamsHelper(params object[] elements) => elements; - private void SumNumbersWithSubtasking(Service sessionService, - int maxNumberToSum = 16, - int subtaskSplitCount = 2) + private void SumNumbersWithSubtasking(ISubmitterService sessionService, + int maxNumberToSum = 16, + int subtaskSplitCount = 2) { Log.LogInformation($"Launching Sum of numbers 1 to {maxNumberToSum}"); var numbers = Enumerable.Range(1, diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClientTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClientTest.cs index 34e96c01..b6e73e9b 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClientTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiClientTest.cs @@ -16,6 +16,7 @@ using System.Linq; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Common; using ArmoniK.EndToEndTests.Common; diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckTypeOfSubmission/CheckAllSubmissionsClient.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckTypeOfSubmission/CheckAllSubmissionsClient.cs index a3e2ae4b..1a5384fd 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckTypeOfSubmission/CheckAllSubmissionsClient.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckTypeOfSubmission/CheckAllSubmissionsClient.cs @@ -66,7 +66,8 @@ public override void EntryPoint() var resultClient = new ArmonikSymphonyClient(Configuration, LoggerFactory); - var resultService = resultClient.OpenSession(sessionService.SessionId); + var resultService = resultClient.OpenSession(sessionService.SessionId, + taskOptions); Log.LogInformation($"New session created : {sessionService}"); diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIAdminTestClient.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIAdminTestClient.cs index 0afc2177..3f5eb874 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIAdminTestClient.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIAdminTestClient.cs @@ -22,9 +22,10 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using ArmoniK.DevelopmentKit.Client.Common.Status; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Factory; using ArmoniK.DevelopmentKit.Client.Unified.Services.Admin; -using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; using ArmoniK.EndToEndTests.Common; @@ -131,8 +132,8 @@ private static object[] ParamsHelper(params object[] elements) /// /// /// - private void RunningAndCancelSession(Service sessionService, - ServiceAdmin serviceAdmin) + private void RunningAndCancelSession(ISubmitterService sessionService, + ServiceAdmin serviceAdmin) { var numbers = new List { diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIClientTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIClientTest.cs index 04c8b59f..c8b05027 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIClientTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPIClientTest.cs @@ -18,6 +18,7 @@ using System.Linq; using ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Common; using NUnit.Framework; diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPITestClient.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPITestClient.cs index 8b42ecdd..063b47b6 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPITestClient.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/CheckUnifiedApi/SimpleUnifiedAPITestClient.cs @@ -21,8 +21,8 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Factory; -using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; using ArmoniK.DevelopmentKit.Common.Extensions; using ArmoniK.EndToEndTests.Common; @@ -159,7 +159,7 @@ private static object[] ParamsHelper(params object[] elements) /// The first test developed to validate dependencies subTasking /// /// - private void ClientStartup1(Service sessionService) + private void ClientStartup1(ISubmitterService sessionService) { var numbers = new List { @@ -207,7 +207,7 @@ private void ClientStartup1(Service sessionService) /// The first test developed to validate dependencies subTasking /// /// - private void ClientStartup2(Service sessionService) + private void ClientStartup2(ISubmitterService sessionService) { var numbers = new List { @@ -249,7 +249,7 @@ private void ClientStartup2(Service sessionService) /// The first test developed to validate dependencies subTasking /// /// - private void ClientStartup3(Service sessionService) + private void ClientStartup3(ISubmitterService sessionService) { var numbers = new List { diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClient.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClient.cs index f0e3adfd..0e02ef9e 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClient.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClient.cs @@ -23,8 +23,8 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.DevelopmentKit.Client.Common; using ArmoniK.DevelopmentKit.Client.Common.Exceptions; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Factory; -using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; using ArmoniK.DevelopmentKit.Common.Extensions; using ArmoniK.EndToEndTests.Common; @@ -160,7 +160,7 @@ await Task.Delay(TimeSpan.FromSeconds(seconds), /// The number of task to submit /// The number of element n x M in the vector /// - private void ComputeVector(Service sessionService, + private void ComputeVector(ISubmitterService sessionService, int nbTasks, int nbElement, CancellationTokenSource cancellationTokenSource) diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClientTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClientTest.cs index 3e2eca96..6af3e09c 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClientTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargePayloadSubmit/LargePayloadSubmitClientTest.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Threading; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Common; using Microsoft.Extensions.Configuration; diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargeSubmitAsync/LargeSubmitAsyncTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargeSubmitAsync/LargeSubmitAsyncTest.cs index fb56192e..8e6600c8 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargeSubmitAsync/LargeSubmitAsyncTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/LargeSubmitAsync/LargeSubmitAsyncTest.cs @@ -93,11 +93,14 @@ public void Check_That_Buffering_With_SubmitAsync_Is_Working(int nbTasks, for (indexTask = 0; indexTask < nbTasks; indexTask++) { - taskIds.Add(service.SubmitAsync("ComputeSum", - UnitTestHelperBase.ParamsHelper(numbers, - workloadTimeInMs), - localUnifiedTestHelper, - token: cancellationSource.Token)); + taskIds.Add(service!.SubmitAsync("ComputeSum", + new object[] + { + numbers, + workloadTimeInMs, + }, + localUnifiedTestHelper, + token: cancellationSource.Token)); } //System.Threading.Thread.Sleep(10000); diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/PayloadIntegrityTestClient/PayloadIntegrityTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/PayloadIntegrityTestClient/PayloadIntegrityTest.cs index db5e934e..e98392dd 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/PayloadIntegrityTestClient/PayloadIntegrityTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/PayloadIntegrityTestClient/PayloadIntegrityTest.cs @@ -24,8 +24,8 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.DevelopmentKit.Client.Common; +using ArmoniK.DevelopmentKit.Client.Common.Submitter; using ArmoniK.DevelopmentKit.Client.Unified.Factory; -using ArmoniK.DevelopmentKit.Client.Unified.Services.Submitter; using ArmoniK.DevelopmentKit.Common; using AutoFixture; @@ -141,8 +141,8 @@ public void CopyPayload(int maxConcurrentBuffers, taskAndData_.Clear(); } - private async Task NewSubmitCallAsync(Fixture fixture, - Service service) + private async Task NewSubmitCallAsync(Fixture fixture, + ISubmitterService service) { var payload = new[] { diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/Priority/Priority.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/Priority/Priority.cs index 1143f3af..05cf810a 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/Priority/Priority.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/Priority/Priority.cs @@ -112,12 +112,9 @@ public void TestThatPrioritiesAreAccountedFor() { var options = properties.TaskOptions.Clone(); options.Priority = t; - var payload = new ArmonikPayload - { - ClientPayload = BitConverter.GetBytes(options.Priority), - MethodName = "GetPriority", - SerializedArguments = true, - }.Serialize(); + var payload = new ArmonikPayload("GetPriority", + BitConverter.GetBytes(options.Priority), + true).Serialize(); foreach (var submitTask in service.SubmitTasks(Enumerable.Repeat(payload, nTasksPerSessionPerPriority), taskOptions: options)) @@ -130,7 +127,7 @@ public void TestThatPrioritiesAreAccountedFor() var results = new List<(string, int, int)>(service.GetResults(tasks.Keys) .Select(tuple => (tuple.Item1, tasks[tuple.Item1], - BitConverter.ToInt32((ProtoSerializer.DeSerializeMessageObjectArray(tuple.Item2)[0] as byte[])!, + BitConverter.ToInt32((ProtoSerializer.Deserialize(tuple.Item2)[0] as byte[])!, 0)))); foreach (var (taskId, expected, actual) in results) diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/ArmoniK.EndToEndTests.Common.csproj b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/ArmoniK.EndToEndTests.Common.csproj index 523eac6a..8fc98d6d 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/ArmoniK.EndToEndTests.Common.csproj +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/ArmoniK.EndToEndTests.Common.csproj @@ -1,9 +1,8 @@ - netstandard2.0;net6.0 + netstandard2.0 enable - enable 1.0.0-700 true @@ -14,11 +13,9 @@ - - diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/TestsContext.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/TestsContext.cs index 913c0669..30d85d69 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/TestsContext.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Common/TestsContext.cs @@ -14,7 +14,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#nullable enable using System.Reflection; namespace ArmoniK.EndToEndTests.Common; diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Dockerfile b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Dockerfile new file mode 100644 index 00000000..a1e3466a --- /dev/null +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Dockerfile @@ -0,0 +1,16 @@ +ARG WORKER_DLL_IMAGE=dockerhubaneo/armonik_worker_dll:0.13.1-01307g2fdaf5b.7.2fdaf5b + +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build +WORKDIR /src +COPY . . + + +WORKDIR "/src/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker" + +RUN dotnet publish --self-contained -c Release -r linux-x64 -f net6.0 . + + +FROM ${WORKER_DLL_IMAGE} AS final + +ENV ServiceAssemblyPath=/guest +COPY --from=build /src/Tests/ArmoniK.EndToEndTests/publish/ArmoniK.EndToEndTests.Worker/1.0.0-700/ /guest diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/AggregationPriority/AggregationPriority.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/AggregationPriority/AggregationPriority.cs index b5380cf8..a77c7097 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/AggregationPriority/AggregationPriority.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/AggregationPriority/AggregationPriority.cs @@ -170,12 +170,9 @@ private static byte[] ToArmoniKPayload(string methodName, object argument) { var payload = JsonSerializer.SerializeToUtf8Bytes(argument); - return new ArmonikPayload - { - MethodName = methodName, - ClientPayload = ProtoSerializer.SerializeMessageObject(payload), - SerializedArguments = false, - }.Serialize(); + return new ArmonikPayload(methodName, + ProtoSerializer.Serialize(payload), + false).Serialize(); } /// @@ -192,7 +189,7 @@ private static TaskResult FromTaskResult(byte[] payload) nameof(payload)); } - var deprot = ProtoSerializer.DeSerializeMessageObjectArray(payload); + var deprot = ProtoSerializer.Deserialize(payload); var taskResult = TaskResult.Deserialize(deprot[0] as byte[]); if (taskResult is null) diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiWorker.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiWorker.cs index 26105e7a..7f4b84ab 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiWorker.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/CheckSubtaskingTreeUnifiedApi/SubtaskingTreeUnifiedApiWorker.cs @@ -159,7 +159,7 @@ public byte[] AggregateValues(byte[] serializedClientPayload) throw new WorkerApiException($"Cannot retrieve result from taskId {TaskContext.DependenciesTaskIds?.Single()}"); } - var deprot = ProtoSerializer.DeSerializeMessageObjectArray(taskDependency.Value); + var deprot = ProtoSerializer.Deserialize(taskDependency.Value); var dependencyResultPayload = ClientPayload.Deserialize(deprot[0] as byte[]); dependencyValues.Add(dependencyResultPayload.Result); aggregatedValuesSum += dependencyResultPayload.Result; diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/LargePayloadSubmit/LargePayloadSubmitWorker.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/LargePayloadSubmit/LargePayloadSubmitWorker.cs index 71b97c4d..d35ad37a 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/LargePayloadSubmit/LargePayloadSubmitWorker.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Worker/Tests/LargePayloadSubmit/LargePayloadSubmitWorker.cs @@ -15,7 +15,6 @@ // limitations under the License. using System; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Threading; @@ -34,8 +33,8 @@ public class LargePayloadSubmitWorker : BaseService /// The first arguments from Client call /// The second arguments from client call /// The result to return - public static double ComputeSum([NotNull] double[] inputs, - int workloadTime) + public static double ComputeSum(double[] inputs, + int workloadTime) { if (inputs == null) { diff --git a/Worker/src/Common/Archive/IArchiver.cs b/Worker/src/Common/Archive/IArchiver.cs index f18f651b..8f157206 100644 --- a/Worker/src/Common/Archive/IArchiver.cs +++ b/Worker/src/Common/Archive/IArchiver.cs @@ -14,8 +14,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using ArmoniK.DevelopmentKit.Common; - namespace ArmoniK.DevelopmentKit.Worker.Common.Archive; /// @@ -26,31 +24,23 @@ public interface IArchiver /// /// Extracts an archive file /// - /// File adapter to fetch the file /// File name - /// Path to assembly file - public string ExtractArchive(IFileAdapter fileAdapter, - string filename); + /// Package Id + /// Overwrite the files if they have been already extracted + /// Path to extracted package folder + public string ExtractArchive(string filename, + PackageId packageId, + bool overwrite = false); /// - /// Checks if the archive has already been extracted + /// Checks if the archive has already been extracted. If the file is being extracted by another process, waits for its + /// completion to return an answer /// - /// File adapter to fetch the file - /// File name - /// Number of 2 seconds intervals to wait for the lock file to be + /// Package Id + /// If the file is being extracted by another process, wait until this timeout + /// Interval between file checks while waiting for extraction /// True if the archive has already been extracted, false otherwise - public bool ArchiveAlreadyExtracted(IFileAdapter fileAdapter, - string fileName, - int waitForArchiver); - - /// - /// Download the archive from the fileAdapter - /// - /// File adapter to fetch the file - /// File name - /// If set to true, doesn't download if the file already exists - /// Path to the download archive - public string DownloadArchive(IFileAdapter fileAdapter, - string filename, - bool skipIfExists); + public bool ArchiveAlreadyExtracted(PackageId packageId, + int waitExtractionTimeoutMs = 60000, + int waitSpinIntervalMs = 1000); } diff --git a/Worker/src/Common/Archive/ZipArchiver.cs b/Worker/src/Common/Archive/ZipArchiver.cs index 9d18565c..7b188c49 100644 --- a/Worker/src/Common/Archive/ZipArchiver.cs +++ b/Worker/src/Common/Archive/ZipArchiver.cs @@ -15,72 +15,61 @@ // limitations under the License. using System; -using System.Collections.Generic; using System.IO; using System.IO.Compression; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; using System.Threading; -using ArmoniK.DevelopmentKit.Common; using ArmoniK.DevelopmentKit.Common.Exceptions; +using ArmoniK.DevelopmentKit.Common.Utils; namespace ArmoniK.DevelopmentKit.Worker.Common.Archive; +/// +/// Class used do handle zip archives +/// public class ZipArchiver : IArchiver { - private const string RootAppPath = "/tmp/packages"; + private readonly string rootAppPath_; - /// - /// Thrown if the dll and the lockfile don't exist - /// Thrown when it has waited too long for the lock file to be liberated - public bool ArchiveAlreadyExtracted(IFileAdapter fileAdapter, - string fileName, - int waitForArchiver = 300) - { - var assemblyInfo = ExtractNameAndVersion(Path.Combine(fileAdapter.DestinationDirPath, - fileName)); - var info = assemblyInfo as string[] ?? assemblyInfo.ToArray(); - var assemblyName = info.ElementAt(0); - var assemblyVersion = info.ElementAt(1); - var basePath = $"{RootAppPath}/{assemblyName}/{assemblyVersion}"; - - if (!Directory.Exists($"{RootAppPath}/{assemblyName}/{assemblyVersion}")) - { - return false; - } + /// + /// Creates a zip archive handler + /// + /// Base path to extract zip files + public ZipArchiver(string assembliesBasePath) + => rootAppPath_ = assembliesBasePath; - //Now at least if dll exists or if a lock file exists and wait for unlock - if (File.Exists($"{basePath}/{assemblyName}.dll")) - { - return true; - } + /// + public bool ArchiveAlreadyExtracted(PackageId packageId, + int waitForExtraction = 60000, + int spinInterval = 1000) + { + var pathToAssemblyDir = Path.Combine(rootAppPath_, + packageId.PackageSubpath); - if (!File.Exists($"{basePath}/{assemblyName}.lock")) + if (!Directory.Exists(pathToAssemblyDir)) { - throw new FileNotFoundException($"Cannot find Service. Assembly name {basePath}/{assemblyName}.dll"); + return false; } - var retry = 0; - const int loopingWait = 2; // 2 secs + var mainAssembly = Path.Combine(pathToAssemblyDir, + packageId.MainAssemblyFileName); - if (waitForArchiver == 0) + if (File.Exists(mainAssembly)) { return true; } - while (!File.Exists($"{basePath}/{assemblyName}.lock")) + var lockFileName = Path.Combine(pathToAssemblyDir, + $"{packageId.ApplicationName}.lock"); + + while (File.Exists(lockFileName) && waitForExtraction > 0) { - Thread.Sleep(loopingWait * 1000); - retry++; - if (retry > waitForArchiver >> 2) - { - throw new WorkerApiException($"Wait for unlock unzip was timeout after {waitForArchiver * loopingWait} seconds"); - } + Thread.Sleep(Math.Min(spinInterval, + waitForExtraction)); + waitForExtraction -= spinInterval; } - return false; + return File.Exists(mainAssembly) && !File.Exists(lockFileName); } /// @@ -88,113 +77,61 @@ public bool ArchiveAlreadyExtracted(IFileAdapter fileAdapter, /// Thrown if the file isn't a zip archive or if the lock file isn't lockable when the /// archive is being extracted by another process /// - public string ExtractArchive(IFileAdapter fileAdapter, - string filename) + public string ExtractArchive(string filename, + PackageId packageId, + bool overwrite = false) { if (!IsZipFile(filename)) { throw new WorkerApiException("Cannot yet extract or manage raw data other than zip archive"); } - var assemblyInfo = ExtractNameAndVersion(filename); - var info = assemblyInfo as string[] ?? assemblyInfo.ToArray(); - var assemblyVersion = info.ElementAt(1); - var assemblyName = info.ElementAt(0); - - - var pathToAssembly = $"{RootAppPath}/{assemblyName}/{assemblyVersion}/{assemblyName}.dll"; - var pathToAssemblyDir = $"{RootAppPath}/{assemblyName}/{assemblyVersion}"; - - if (ArchiveAlreadyExtracted(fileAdapter, - filename, - 20)) - { - return pathToAssembly; - } + var pathToAssemblyDir = Path.Combine(rootAppPath_, + packageId.PackageSubpath); + var mainAssembly = Path.Combine(pathToAssemblyDir, + packageId.MainAssemblyFileName); if (!Directory.Exists(pathToAssemblyDir)) { Directory.CreateDirectory(pathToAssemblyDir); } - var lockFileName = $"{pathToAssemblyDir}/{assemblyName}.lock"; + var lockFileName = Path.Combine(pathToAssemblyDir, + $"{packageId.ApplicationName}.lock"); - - using (var fileStream = new FileStream(lockFileName, - FileMode.OpenOrCreate, - FileAccess.ReadWrite, - FileShare.ReadWrite)) + using (var spinLock = new FileSpinLock(lockFileName, + timeoutMs: 60000)) { - var lockfileForExtractionString = "Lockfile for extraction"; - - var unicodeEncoding = new UnicodeEncoding(); - var textLength = unicodeEncoding.GetByteCount(lockfileForExtractionString); - - if (fileStream.Length == 0) - //Try to lock file to protect extraction + if (spinLock.HasLock) { - fileStream.Write(new UnicodeEncoding().GetBytes(lockfileForExtractionString), - 0, - unicodeEncoding.GetByteCount(lockfileForExtractionString)); + if (overwrite || !File.Exists(mainAssembly)) + { + try + { + ZipFile.ExtractToDirectory(filename, + rootAppPath_); + } + catch (Exception e) + { + throw new WorkerApiException($"Could not extract zip file {filename}", + e); + } + } } - - try - { - fileStream.Lock(0, - textLength); - } - catch (IOException) - { - return pathToAssembly; - } - catch (Exception e) - { - throw new WorkerApiException(e); - } - - - try + else { - ZipFile.ExtractToDirectory(Path.Combine(fileAdapter.DestinationDirPath, - filename), - RootAppPath); - } - catch (Exception e) - { - throw new WorkerApiException(e); - } - finally - { - fileStream.Unlock(0, - textLength); + throw new WorkerApiException($"Could not lock file to extract zip {filename}"); } } - File.Delete(lockFileName); - //Check now if the assembly is present - if (!File.Exists(pathToAssembly)) + if (!File.Exists(mainAssembly)) { - throw new WorkerApiException($"Fail to find assembly {pathToAssembly}. Something went wrong during the extraction. " + - $"Please sure that tree folder inside is {assemblyName}/{assemblyVersion}/*.dll"); + throw new WorkerApiException($"Fail to find assembly {mainAssembly}. Something went wrong during the extraction. " + + $"Please make sure that the folder tree inside the zip file is {packageId.ApplicationName}/{packageId.ApplicationVersion}/*.dll"); } - return pathToAssembly; - } - - /// - public string DownloadArchive(IFileAdapter fileAdapter, - string fileName, - bool skipIfExists = true) - { - if (!skipIfExists || !File.Exists(Path.Combine(fileAdapter.DestinationDirPath, - fileName))) - { - return fileAdapter.DownloadFile(fileName); - } - - return Path.Combine(fileAdapter.DestinationDirPath, - fileName); + return pathToAssemblyDir; } /// @@ -208,45 +145,4 @@ public static bool IsZipFile(string assemblyNameFilePath) var extension = Path.GetExtension(assemblyNameFilePath); return extension?.ToLower() == ".zip"; } - - /// - /// - /// - /// - /// - public static IEnumerable ExtractNameAndVersion(string assemblyNameFilePath) - { - string filePathNoExt; - - try - { - filePathNoExt = Path.GetFileNameWithoutExtension(assemblyNameFilePath); - } - catch (ArgumentException e) - { - throw new WorkerApiException(e); - } - - var groups = Regex.Split(filePathNoExt, - "-v", - RegexOptions.IgnoreCase); - - if (groups.Length == 2) - { - return groups; - } - - throw new WorkerApiException($"File name \"{filePathNoExt}\" format doesn't match: {(groups.Length > 2 ? "too many versions." : "no version specified.")}"); - } - - public static string GetLocalPathToAssembly(string pathToZip) - { - var assemblyInfo = ExtractNameAndVersion(pathToZip); - var info = assemblyInfo as string[] ?? assemblyInfo.ToArray(); - var assemblyName = info.ElementAt(0); - var assemblyVersion = info.ElementAt(1); - var basePath = $"{RootAppPath}/{assemblyName}/{assemblyVersion}"; - - return $"{basePath}/{assemblyName}.dll"; - } } diff --git a/Worker/src/Common/ArmoniK.DevelopmentKit.Worker.Common.csproj b/Worker/src/Common/ArmoniK.DevelopmentKit.Worker.Common.csproj index 139e1e86..6416bed1 100644 --- a/Worker/src/Common/ArmoniK.DevelopmentKit.Worker.Common.csproj +++ b/Worker/src/Common/ArmoniK.DevelopmentKit.Worker.Common.csproj @@ -10,7 +10,6 @@ - diff --git a/Worker/src/Common/IAppsLoader.cs b/Worker/src/Common/IAppsLoader.cs index 7e29b1b4..12dea21a 100644 --- a/Worker/src/Common/IAppsLoader.cs +++ b/Worker/src/Common/IAppsLoader.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2023. All rights reserved. // diff --git a/Worker/src/Common/ServiceId.cs b/Worker/src/Common/ServiceId.cs new file mode 100644 index 00000000..9d56668d --- /dev/null +++ b/Worker/src/Common/ServiceId.cs @@ -0,0 +1,182 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2023. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; + +using ArmoniK.DevelopmentKit.Common; + +namespace ArmoniK.DevelopmentKit.Worker.Common; + +/// +/// Identifier for a Package +/// +public readonly struct PackageId : IEquatable +{ + /// + /// Creates a PackageId + /// + /// Application name + /// Application version + public PackageId(string applicationName, + string applicationVersion) + { + ApplicationName = applicationName; + ApplicationVersion = applicationVersion; + } + + /// + /// Application name + /// + public string ApplicationName { get; } + + /// + /// Application version + /// + public string ApplicationVersion { get; } + + /// + public bool Equals(PackageId other) + => ApplicationName == other.ApplicationName && ApplicationVersion == other.ApplicationVersion; + + /// + /// Indicates whether the objects are equal. + /// + /// Object a + /// Object b + /// true if the objects are equal, false otherwise + public static bool operator ==(PackageId a, + PackageId b) + => a.Equals(b); + + /// + /// Indicates whether the objects are not equal. + /// + /// Object a + /// Object b + /// false if the objects are equal, true otherwise + public static bool operator !=(PackageId a, + PackageId b) + => !(a == b); + + /// + public override bool Equals(object obj) + => obj is PackageId id && Equals(id); + + /// + public override int GetHashCode() + => HashCode.Combine(ApplicationName, + ApplicationVersion); + + /// + /// Name of this package's main assembly file + /// + public string MainAssemblyFileName + => $"{ApplicationName}.dll"; + + /// + /// Subpath of this package + /// + public string PackageSubpath + => Path.Combine(ApplicationName, + ApplicationVersion); + + /// + /// Name of the zip file for that package + /// + public string ZipFileName + => $"{ApplicationName}-v{ApplicationVersion}.zip"; + + + /// + public override string ToString() + => $"{ApplicationName}-v{ApplicationVersion}"; +} + +/// +/// Identifier for a Service +/// +public readonly struct ServiceId : IEquatable +{ + /// + /// Creates a ServiceId + /// + /// PackageId + /// Namespace + /// Engine type + public ServiceId(PackageId packageId, + string applicationNamespace, + EngineType engineType) + { + PackageId = packageId; + ApplicationNamespace = applicationNamespace; + EngineType = engineType; + } + + /// + /// PackageId of the service + /// + public PackageId PackageId { get; } + + /// + /// Namespace of the service + /// + public string ApplicationNamespace { get; } + + /// + /// Type of engine of the service + /// + public EngineType EngineType { get; } + + + /// + public bool Equals(ServiceId other) + => EngineType == other.EngineType && ApplicationNamespace == other.ApplicationNamespace && PackageId.Equals(other.PackageId); + + /// + /// Indicates whether the objects are equal. + /// + /// Object a + /// Object b + /// true if the objects are equal, false otherwise + public static bool operator ==(ServiceId a, + ServiceId b) + => a.Equals(b); + + /// + /// Indicates whether the objects are not equal. + /// + /// Object a + /// Object b + /// false if the objects are equal, true otherwise + public static bool operator !=(ServiceId a, + ServiceId b) + => !(a == b); + + /// + public override bool Equals(object obj) + => obj is ServiceId && Equals(obj); + + /// + public override int GetHashCode() + => HashCode.Combine(PackageId, + ApplicationNamespace, + (int)EngineType); + + /// + public override string ToString() + => $"{PackageId}#{EngineType}#{ApplicationNamespace}"; +} diff --git a/Worker/src/DLLWorker/AddonsAssemblyLoadConext.cs b/Worker/src/DLLWorker/AddOnAssemblyLoadContext.cs similarity index 75% rename from Worker/src/DLLWorker/AddonsAssemblyLoadConext.cs rename to Worker/src/DLLWorker/AddOnAssemblyLoadContext.cs index ce48f476..f908c59b 100644 --- a/Worker/src/DLLWorker/AddonsAssemblyLoadConext.cs +++ b/Worker/src/DLLWorker/AddOnAssemblyLoadContext.cs @@ -20,37 +20,37 @@ namespace ArmoniK.DevelopmentKit.Worker.DLLWorker; -internal class AddonsAssemblyLoadContext : AssemblyLoadContext +internal class AddOnAssemblyLoadContext : AssemblyLoadContext { - private readonly AssemblyDependencyResolver _resolver; - private readonly AssemblyDependencyResolver _rootResolver; + private readonly AssemblyDependencyResolver resolver_; + private readonly AssemblyDependencyResolver rootResolver_; - public AddonsAssemblyLoadContext() + public AddOnAssemblyLoadContext() : base(Guid.NewGuid() .ToString(), true) { } - public AddonsAssemblyLoadContext(string mainAssemblyToLoadPath) + public AddOnAssemblyLoadContext(string mainAssemblyToLoadPath) : base(Guid.NewGuid() .ToString(), true) { - _rootResolver = new AssemblyDependencyResolver(Assembly.GetExecutingAssembly() + rootResolver_ = new AssemblyDependencyResolver(Assembly.GetExecutingAssembly() .Location); - _resolver = new AssemblyDependencyResolver(mainAssemblyToLoadPath); + resolver_ = new AssemblyDependencyResolver(mainAssemblyToLoadPath); } protected override Assembly Load(AssemblyName name) { - if (_rootResolver.ResolveAssemblyToPath(name) != null) + if (rootResolver_.ResolveAssemblyToPath(name) != null) { return null; } - var assemblyPath = _resolver.ResolveAssemblyToPath(name); + var assemblyPath = resolver_.ResolveAssemblyToPath(name); return assemblyPath != null ? LoadFromAssemblyPath(assemblyPath) @@ -65,7 +65,7 @@ protected override Assembly Load(AssemblyName name) /// A handle to the loaded library, or . protected override IntPtr LoadUnmanagedDll(string unmanagedDllName) { - var assemblyPath = _resolver.ResolveUnmanagedDllToPath(unmanagedDllName); + var assemblyPath = resolver_.ResolveUnmanagedDllToPath(unmanagedDllName); return assemblyPath != null ? LoadUnmanagedDllFromPath(assemblyPath) : IntPtr.Zero; diff --git a/Worker/src/DLLWorker/ApplicationPackageManager.cs b/Worker/src/DLLWorker/ApplicationPackageManager.cs new file mode 100644 index 00000000..316cf497 --- /dev/null +++ b/Worker/src/DLLWorker/ApplicationPackageManager.cs @@ -0,0 +1,160 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2023. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.IO; +using System.Linq; + +using ArmoniK.DevelopmentKit.Common; +using ArmoniK.DevelopmentKit.Common.Exceptions; +using ArmoniK.DevelopmentKit.Worker.Common; +using ArmoniK.DevelopmentKit.Worker.Common.Adapter; +using ArmoniK.DevelopmentKit.Worker.Common.Archive; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.DevelopmentKit.Worker.DLLWorker; + +/// +/// Manages application packages +/// +public class ApplicationPackageManager +{ + private readonly string archivePath_; + private readonly IArchiver archiver_; + private readonly string assembliesSearchPath_; + private readonly IFileAdapter fileAdapter_; + private readonly ILogger logger_; + + /// + /// Creates an application package manager + /// + /// DLLWorker configuration + /// Logger factory + /// Thrown when the FileStorageType is unspecified in the configuration + public ApplicationPackageManager(IConfiguration configuration, + ILoggerFactory loggerFactory) + { + assembliesSearchPath_ = configuration[AppsOptions.GridAssemblyPathKey] ?? "/tmp/assemblies"; + archivePath_ = configuration[AppsOptions.GridZipVolumePath] ?? "/data"; + switch (configuration["FileStorageType"]) + { + case "FS": + fileAdapter_ = new FsAdapter(configuration[AppsOptions.GridDataVolumesKey] ?? "/data", + archivePath_); + break; + case "S3": + { + var configurationSection = configuration.GetSection("S3Storage"); + fileAdapter_ = new S3Adapter(configurationSection["ServiceURL"], + configurationSection["BucketName"], + configurationSection["AccessKeyId"], + configurationSection["SecretAccessKey"], + "", + configurationSection.GetValue("MustForcePathStyle", + false), + archivePath_); + break; + } + default: + throw new WorkerApiException("Cannot find the FileStorageType in the IConfiguration. Please make sure you have properly set the field [FileStorageType]"); + } + + archiver_ = new ZipArchiver(assembliesSearchPath_); + logger_ = loggerFactory.CreateLogger(); + } + + /// + /// Loads the application package. If the package is already loaded just returns its base path. + /// + /// Package Id + /// Path to the application package + [CanBeNull] + public string LoadApplicationPackage(PackageId packageId) + { + var localFile = GetApplicationAssemblyFile(packageId, + packageId.MainAssemblyFileName); + if (localFile != null) + { + logger_.LogDebug("Package {packageId} is already loaded", + packageId); + // Package is already loaded + return Path.GetDirectoryName(localFile); + } + + // Try to get the local zip, download it if it doesn't exist + var localZip = GetLocalApplicationZip(packageId) ?? fileAdapter_.DownloadFile(packageId.ZipFileName); + + if (!archiver_.ArchiveAlreadyExtracted(packageId)) + { + logger_.LogInformation("Extracting {packageId} from archive {localZip}", + packageId, + localZip); + var extractedPath = archiver_.ExtractArchive(localZip, + packageId); + logger_.LogInformation("Package {packageId} successfully extracted from {localZip}", + packageId, + localZip); + return extractedPath; + } + + // Get the directory where the main assembly is located + return Path.GetDirectoryName(GetApplicationAssemblyFile(packageId, + packageId.MainAssemblyFileName)); + } + + /// + /// Get the path to the given assembly of the package + /// + /// PackageId + /// Name of the assembly + /// + /// List of search paths for the assembly, defaults to the usual package location if not + /// specified + /// + /// Path to the assembly in the package, null if it cannot be found + [CanBeNull] + public string GetApplicationAssemblyFile(PackageId packageId, + string assemblyName, + [CanBeNull] string[] searchPaths = null) + => (searchPaths?.AsEnumerable() + .Select(path => Path.Combine(path, + assemblyName)) ?? new[] + { + Path.Combine(assembliesSearchPath_, + assemblyName), + Path.Combine(assembliesSearchPath_, + packageId.PackageSubpath, + assemblyName), + }).FirstOrDefault(File.Exists); + + /// + /// Get the path to the local zip file for the package + /// + /// PackageId + /// Path to the zip file, null if it cannot be found + [CanBeNull] + private string GetLocalApplicationZip(PackageId packageId) + { + var zipPath = Path.Combine(archivePath_, + packageId.ZipFileName); + return File.Exists(zipPath) + ? zipPath + : null; + } +} diff --git a/Worker/src/DLLWorker/AppsLoader.cs b/Worker/src/DLLWorker/AppsLoader.cs index 7a07cc18..fc82969b 100644 --- a/Worker/src/DLLWorker/AppsLoader.cs +++ b/Worker/src/DLLWorker/AppsLoader.cs @@ -22,7 +22,6 @@ using ArmoniK.DevelopmentKit.Common; using ArmoniK.DevelopmentKit.Common.Exceptions; using ArmoniK.DevelopmentKit.Worker.Common; -using ArmoniK.DevelopmentKit.Worker.Common.Archive; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -37,36 +36,27 @@ public class AppsLoader : IAppsLoader private readonly ILogger logger_; private Assembly assembly_; - public AppsLoader(IConfiguration configuration, - ILoggerFactory loggerFactory, - string engineTypeAssemblyName, - IFileAdapter fileAdapter, - string fileName) + public AppsLoader(ApplicationPackageManager packageManager, + ILoggerFactory loggerFactory, + string engineTypeAssemblyName, + PackageId packageId) { engineType_ = EngineTypeHelper.ToEnum(engineTypeAssemblyName); - FileAdapter = fileAdapter; - ArmoniKDevelopmentKitServerApi = new EngineTypes()[engineType_]; logger_ = loggerFactory.CreateLogger(); - var archiver = new ZipArchiver(); - - if (!archiver.ArchiveAlreadyExtracted(fileAdapter, - fileName)) - { - archiver.DownloadArchive(fileAdapter, - fileName); - archiver.ExtractArchive(fileAdapter, - fileName); - } + var localAssemblySearchPath = packageManager.LoadApplicationPackage(packageId) ?? throw new WorkerApiException($"Could not load package {packageId.PackageSubpath}"); + var localPathToAssembly = packageManager.GetApplicationAssemblyFile(packageId, + packageId.MainAssemblyFileName, + new[] + { + localAssemblySearchPath, + }) ?? + throw new WorkerApiException($"Could not find main assembly {localAssemblySearchPath}/{packageId.MainAssemblyFileName}"); - - var localPathToAssembly = ZipArchiver.GetLocalPathToAssembly(Path.Combine(fileAdapter.DestinationDirPath, - fileName)); - - UserAssemblyLoadContext = new AddonsAssemblyLoadContext(localPathToAssembly); + UserAssemblyLoadContext = new AddOnAssemblyLoadContext(localPathToAssembly); try { @@ -80,7 +70,12 @@ public AppsLoader(IConfiguration configuration, PathToAssembly = localPathToAssembly; - var localPathToAssemblyGridWorker = $"{Path.GetDirectoryName(localPathToAssembly)}/{ArmoniKDevelopmentKitServerApi}.dll"; + var localPathToAssemblyGridWorker = packageManager.GetApplicationAssemblyFile(packageId, + $"{ArmoniKDevelopmentKitServerApi}.dll", + new[] + { + localAssemblySearchPath, + }); try { @@ -143,8 +138,6 @@ Assembly LoadFromSameFolder(object sender, public IConfiguration Configuration { get; } - public IFileAdapter FileAdapter { get; set; } - public string PathToAssembly { get; set; } public string PathToAssemblyGridWorker { get; set; } @@ -239,16 +232,4 @@ public T GetServiceContainerInstance(string appNamespace, ~AppsLoader() => Dispose(); - - public bool RequestNewAssembly(string engineType, - string pathToZipFile) - { - if (pathToZipFile == null) - { - throw new ArgumentNullException(nameof(pathToZipFile), - "pathToZipFile is a null argument"); - } - - return engineType == null || engineType_ != EngineTypeHelper.ToEnum(engineType) || FileAdapter == null || !pathToZipFile.Equals(FileAdapter); - } } diff --git a/Worker/src/DLLWorker/ArmoniK.DevelopmentKit.Worker.DLLWorker.csproj b/Worker/src/DLLWorker/ArmoniK.DevelopmentKit.Worker.DLLWorker.csproj index 4d710097..f7eb987a 100644 --- a/Worker/src/DLLWorker/ArmoniK.DevelopmentKit.Worker.DLLWorker.csproj +++ b/Worker/src/DLLWorker/ArmoniK.DevelopmentKit.Worker.DLLWorker.csproj @@ -6,16 +6,6 @@ true - - Embedded - true - DEBUG;TRACE - - - - true - - diff --git a/Worker/src/DLLWorker/ServiceRequestContext.cs b/Worker/src/DLLWorker/ServiceRequestContext.cs index c66144e4..8bcdbdea 100644 --- a/Worker/src/DLLWorker/ServiceRequestContext.cs +++ b/Worker/src/DLLWorker/ServiceRequestContext.cs @@ -15,14 +15,12 @@ // limitations under the License. using System; -using System.IO; using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.Worker.Worker; using ArmoniK.DevelopmentKit.Common; using ArmoniK.DevelopmentKit.Common.Exceptions; using ArmoniK.DevelopmentKit.Worker.Common; -using ArmoniK.DevelopmentKit.Worker.Common.Adapter; using JetBrains.Annotations; @@ -31,82 +29,6 @@ namespace ArmoniK.DevelopmentKit.Worker.DLLWorker; -public class ServiceId : IEquatable -{ - public ServiceId(string engineTypeName, - string pathToZipFile, - string namespaceService) - => Key = $"{engineTypeName}#{pathToZipFile}#{namespaceService}".ToLower(); - - public string Key { get; } - - /// - public bool Equals(ServiceId other) - { - if (other is null) - { - return false; - } - - if (ReferenceEquals(this, - other)) - { - return true; - } - - return Key == other.Key; - } - - /// Returns a string that represents the current object. - /// A string that represents the current object. - public override string ToString() - => Key; - - /// - public override bool Equals(object obj) - { - if (obj is null) - { - return false; - } - - if (ReferenceEquals(this, - obj)) - { - return true; - } - - return obj.GetType() == GetType() && Equals((ServiceId)obj); - } - - /// - public override int GetHashCode() - => Key != null - ? Key.GetHashCode() - : 0; - - - /// - /// Checks if both ServiceIds are equal - /// - /// ServiceId a - /// ServiceId b - /// Same as a.Equals(b) - public static bool operator ==(ServiceId a, - ServiceId b) - => a?.Equals(b) ?? false; - - /// - /// Checks if both ServiceIds are different - /// - /// ServiceId a - /// ServiceId b - /// Same as !a.Equals(b) - public static bool operator !=(ServiceId a, - ServiceId b) - => !(a == b); -} - public class ArmonikServiceWorker : IDisposable { public ArmonikServiceWorker() @@ -238,38 +160,37 @@ public bool IsNewSessionId(string sessionId) return IsNewSessionId(currentSessionId); } - public ArmonikServiceWorker CreateOrGetArmonikService(IConfiguration configuration, - string engineTypeName, - IFileAdapter fileAdapter, - string fileName, - TaskOptions requestTaskOptions) + public ArmonikServiceWorker CreateOrGetArmonikService(IConfiguration configuration, + ApplicationPackageManager appPackageManager, + string engineTypeName, + PackageId packageId, + TaskOptions requestTaskOptions) { if (string.IsNullOrEmpty(requestTaskOptions.ApplicationNamespace)) { throw new WorkerApiException("Cannot find namespace service in TaskOptions. Please set the namespace"); } - var serviceId = GenerateServiceId(engineTypeName, - Path.Combine(fileAdapter.DestinationDirPath, - fileName), - requestTaskOptions.ApplicationNamespace); + var serviceId = new ServiceId(packageId, + requestTaskOptions.ApplicationNamespace, + EngineTypeHelper.ToEnum(engineTypeName)); if (currentService_?.ServiceId == serviceId) { return currentService_; } - logger_.LogInformation($"Worker needs to load new context, from {currentService_?.ServiceId?.ToString() ?? "null"} to {serviceId}"); + logger_.LogInformation($"Worker needs to load new context, from {currentService_?.ServiceId.ToString() ?? "null"} to {serviceId}"); currentService_?.DestroyService(); currentService_?.Dispose(); currentService_ = null; - var appsLoader = new AppsLoader(configuration, + + var appsLoader = new AppsLoader(appPackageManager, LoggerFactory, engineTypeName, - fileAdapter, - fileName); + packageId); currentService_ = new ArmonikServiceWorker { @@ -284,35 +205,4 @@ public ArmonikServiceWorker CreateOrGetArmonikService(IConfiguration configurati return currentService_; } - - public static ServiceId GenerateServiceId(string engineTypeName, - string uniqueKey, - string namespaceService) - => new(engineTypeName, - uniqueKey, - namespaceService); - - public static IFileAdapter CreateOrGetFileAdapter(IConfiguration configuration, - string localDirectoryZip) - { - var sectionStorage = configuration.GetSection("FileStorageType"); - if (sectionStorage.Exists() && configuration["FileStorageType"] == "FS") - { - return new FsAdapter(localDirectoryZip); - } - - if ((sectionStorage.Exists() && configuration["FileStorageType"] == "S3") || !sectionStorage.Exists()) - { - var configurationSection = configuration.GetSection("S3Storage"); - return new S3Adapter(configurationSection["ServiceURL"], - configurationSection["BucketName"], - configurationSection["AccessKeyId"], - configurationSection["SecretAccessKey"], - "", - configurationSection.GetValue("MustForcePathStyle", - false)); - } - - throw new WorkerApiException("Cannot find the FileStorageType in the IConfiguration. Please make sure you have properly set the field [FileStorageType]"); - } } diff --git a/Worker/src/DLLWorker/Services/ComputerService.cs b/Worker/src/DLLWorker/Services/ComputerService.cs index e34d7dd0..da1c8318 100644 --- a/Worker/src/DLLWorker/Services/ComputerService.cs +++ b/Worker/src/DLLWorker/Services/ComputerService.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Worker.Worker; using ArmoniK.DevelopmentKit.Common; using ArmoniK.DevelopmentKit.Common.Exceptions; +using ArmoniK.DevelopmentKit.Worker.Common; using Grpc.Core; @@ -36,6 +37,8 @@ namespace ArmoniK.DevelopmentKit.Worker.DLLWorker.Services; public class ComputerService : WorkerStreamWrapper { + private readonly ApplicationPackageManager appPackageManager_; + public ComputerService(IConfiguration configuration, GrpcChannelProvider provider, ServiceRequestContext serviceRequestContext) @@ -45,6 +48,8 @@ public ComputerService(IConfiguration configuration, Configuration = configuration; Logger = serviceRequestContext.LoggerFactory.CreateLogger(); ServiceRequestContext = serviceRequestContext; + appPackageManager_ = new ApplicationPackageManager(configuration, + serviceRequestContext.LoggerFactory); Logger.LogDebug("Starting worker...OK"); } @@ -91,21 +96,18 @@ public override async Task Process(ITaskHandler taskHandler) throw new WorkerApiException($"Error in TaskOptions : One of Keys is missing [{string.Join(";", missingKeys.Select(el => $"{el.Item1} => {el.Item2}"))}]"); } - var fileName = $"{taskHandler.TaskOptions.ApplicationName}-v{taskHandler.TaskOptions.ApplicationVersion}.zip"; - var localDirectoryZip = $"{Configuration[AppsOptions.GridDataVolumesKey]}"; + var packageId = new PackageId(taskHandler.TaskOptions.ApplicationName, + taskHandler.TaskOptions.ApplicationVersion); var engineTypeName = string.IsNullOrEmpty(taskHandler.TaskOptions.EngineType) ? EngineType.Symphony.ToString() : taskHandler.TaskOptions.EngineType; - var fileAdapter = ServiceRequestContext.CreateOrGetFileAdapter(Configuration, - localDirectoryZip); - var serviceWorker = ServiceRequestContext.CreateOrGetArmonikService(Configuration, + appPackageManager_, engineTypeName, - fileAdapter, - fileName, + packageId, taskHandler.TaskOptions); diff --git a/Worker/src/Unified/ArmoniK.DevelopmentKit.Worker.Unified.csproj b/Worker/src/Unified/ArmoniK.DevelopmentKit.Worker.Unified.csproj index 1010f856..c19395bf 100644 --- a/Worker/src/Unified/ArmoniK.DevelopmentKit.Worker.Unified.csproj +++ b/Worker/src/Unified/ArmoniK.DevelopmentKit.Worker.Unified.csproj @@ -5,6 +5,7 @@ Library True true + enable diff --git a/Worker/src/Unified/BaseService.cs b/Worker/src/Unified/BaseService.cs index 12e3e351..9b2511b8 100644 --- a/Worker/src/Unified/BaseService.cs +++ b/Worker/src/Unified/BaseService.cs @@ -49,8 +49,8 @@ public abstract class BaseService public BaseService() { Configuration = WorkerHelpers.GetDefaultConfiguration(); - Logger = WorkerHelpers.GetDefaultLoggerFactory(Configuration) - .CreateLogger(); + LoggerFactory = WorkerHelpers.GetDefaultLoggerFactory(Configuration); + Logger = LoggerFactory.CreateLogger(); } /// @@ -185,7 +185,7 @@ public void OnDestroyService(ServiceContext serviceContext) /// public IEnumerable SubmitTasks(IEnumerable payloads, int maxRetries = 5, - TaskOptions taskOptions = null) + TaskOptions? taskOptions = null) => SessionService.SubmitTasks(payloads, maxRetries, taskOptions); diff --git a/Worker/src/Unified/GridWorker.cs b/Worker/src/Unified/GridWorker.cs index c437b422..0889a988 100644 --- a/Worker/src/Unified/GridWorker.cs +++ b/Worker/src/Unified/GridWorker.cs @@ -168,7 +168,7 @@ public byte[] Execute(ITaskHandler taskHandler) { armonikPayload.ClientPayload, } - : ProtoSerializer.DeSerializeMessageObjectArray(armonikPayload.ClientPayload); + : ProtoSerializer.Deserialize(armonikPayload.ClientPayload); MethodInfo methodInfo; if (arguments == null || arguments.Any() == false) @@ -204,7 +204,7 @@ public byte[] Execute(ITaskHandler taskHandler) if (methodInfo == null) { throw new - WorkerApiException($"Cannot found method [{methodName}({string.Join(", ", arguments.Select(x => x.GetType().Name))})] in Service class [{GridAppNamespace}.{GridServiceName}]"); + WorkerApiException($"Cannot find method [{methodName}({string.Join(", ", arguments.Select(x => x.GetType().Name))})] in Service class [{GridAppNamespace}.{GridServiceName}]"); } try @@ -213,10 +213,10 @@ public byte[] Execute(ITaskHandler taskHandler) arguments); if (result != null) { - return new ProtoSerializer().SerializeMessageObjectArray(new[] - { - result, - }); + return ProtoSerializer.Serialize(new[] + { + result, + }); } } // Catch all exceptions from MethodBase.Invoke except TargetInvocationException (triggered by an exception in the invoked code) diff --git a/Worker/src/Unified/TaskWorkerService.cs b/Worker/src/Unified/TaskWorkerService.cs index b6937b10..18db79f4 100644 --- a/Worker/src/Unified/TaskWorkerService.cs +++ b/Worker/src/Unified/TaskWorkerService.cs @@ -248,13 +248,9 @@ public string SubmitTask(string methodName, int maxRetries = 5, TaskOptions taskOptions = null) { - var protoSerializer = new ProtoSerializer(); - ArmonikPayload armonikPayload = new() - { - MethodName = methodName, - ClientPayload = protoSerializer.SerializeMessageObjectArray(arguments), - SerializedArguments = false, - }; + ArmonikPayload armonikPayload = new(methodName, + ProtoSerializer.Serialize(arguments), + false); return SessionService.SubmitTasks(new[] { armonikPayload.Serialize(), @@ -314,13 +310,9 @@ public string SubmitTaskWithDependencies(string methodName, int maxRetries = 5, TaskOptions taskOptions = null) { - var protoSerializer = new ProtoSerializer(); - ArmonikPayload armonikPayload = new() - { - MethodName = methodName, - ClientPayload = protoSerializer.SerializeMessageObjectArray(arguments), - SerializedArguments = false, - }; + ArmonikPayload armonikPayload = new(methodName, + ProtoSerializer.Serialize(arguments), + false); return SessionService.SubmitTasksWithDependencies(new[] { Tuple.Create(armonikPayload.Serialize(), diff --git a/tools/logs2seq.py b/tools/logs2seq.py new file mode 100644 index 00000000..9fccc1c8 --- /dev/null +++ b/tools/logs2seq.py @@ -0,0 +1,115 @@ +from typing import IO +import requests +import json +import argparse +import tarfile +import logging +from pathlib import Path +import sys +import os +import boto3 + + +logger = logging.getLogger(Path(__file__).name) +logging.basicConfig( + level=logging.INFO +) + + +def is_valid_file(valid_log_files: list[str], name: str) -> bool: + """ + Select a specific file or folder based on name + + Args: + valid_log_files: List of valid files + name: The file or folder name to be checked + + Returns: + bool: True if the file name ends with ".log" and match with the names + defined as valid, False otherwise + """ + return any(([name.endswith(".log") and log_file in name for log_file in valid_log_files])) + + +def make_post_request(url: str, data: str): + """ + Make a POST request to the URL of the data + + Args: + url: The URL to which the POST request will be made + valid_log_files: List of valid files + data: The data to be sent in the POST request body + """ + logger.debug(f"send : {len(data)} bytes") + requests.post(url, data=data) + + +def send_log_file(file: str, url: str): + """ + Send log data from a file to a seq URL + + Args: + file: The file object containing log data + url: The URL to which log data will be sent + """ + ctr = 0 + tosend = b"" + for line in file: + line = line.decode("utf-8").strip() + if line.startswith("{"): + json_data = json.loads(line) + if "@t" not in json_data.get("log"): + continue + ctr = ctr + 1 + log_message = bytes(json_data.get("log"), "utf-8") + if len(tosend) + len(log_message) > 100000: + make_post_request(url, tosend) + tosend = log_message + else: + tosend += log_message + logger.info(f"sent : {ctr}") + if tosend != b"": + make_post_request(url, tosend) + + +def extract_jsontar_log(url: str, valid_log_files: list[str], file_name: str): + """ + Extract log data from a tar archive and send to a seq server + + Args: + url : The URL to which log data will be sent + file_name : The name of the tar archive + """ + with tarfile.open(file_name, "r") as file_obj: + for file in file_obj.getnames(): + if is_valid_file(valid_log_files, file): + send_log_file(file_obj.extractfile(file), url) + + +def main(): + parser = argparse.ArgumentParser(description="Download ArmoniK logs in tar format from S3 bucket then send them to Seq.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("bucket_name", help="S3 bucket", type=str) + parser.add_argument("folder_name", help="Folder where extcsharp logs are located", type=str) + parser.add_argument("run_number", help="GitHub workflow run_number", type=str) + parser.add_argument("run_attempt", help="GitHub workflow run_attempt", type=str) + parser.add_argument("file_name", help="file to download from the bucket", type=str) + parser.add_argument("--url", dest="url", help="Seq url", type=str, default="http://localhost:9341/api/events/raw?clef") + parser.add_argument("--files", dest="valid_log_files", nargs='+', help="List of choosen files", default=['control-plane','compute-plane']) + args = parser.parse_args() + + tmp_dir = "./tmp/" + dir_name = os.path.join( + args.folder_name, args.run_number, args.run_attempt) + obj_name = os.path.join(dir_name, args.file_name) + file_name = os.path.join(tmp_dir, obj_name) + + os.makedirs(os.path.join(tmp_dir, dir_name), exist_ok=True) + + s3 = boto3.client('s3') + s3.download_file(args.bucket_name, obj_name, file_name) + + extract_jsontar_log(args.url, args.valid_log_files, file_name) + + +if __name__ == "__main__": + main() diff --git a/tools/parameters.tfvars b/tools/parameters.tfvars new file mode 100644 index 00000000..b77fbdf2 --- /dev/null +++ b/tools/parameters.tfvars @@ -0,0 +1,322 @@ +# Logging level +logging_level = "Information" + +# Uncomment to deploy metrics server +#metrics_server = {} + +# Object storage +# Uncomment either the `redis` or the `minio` parameter +redis = {} +#minio = {} + +# Uncomment this to have minio S3 enabled instead of hostpath shared_storage +#minio_s3_fs = {} # Shared storage + +metrics_exporter = { + extra_conf = { + MongoDB__AllowInsecureTls = true + Serilog__MinimumLevel = "Information" + MongoDB__TableStorage__PollingDelayMin = "00:00:01" + MongoDB__TableStorage__PollingDelayMax = "00:00:10" + } +} + +/*parition_metrics_exporter = { + extra_conf = { + MongoDB__AllowInsecureTls = true + Serilog__MinimumLevel = "Information" + MongoDB__TableStorage__PollingDelayMin = "00:00:01" + MongoDB__TableStorage__PollingDelayMax = "00:00:10" + } +}*/ + +# Parameters of control plane +control_plane = { + limits = { + cpu = "1000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + default_partition = "default" +} + +# Parameters of admin GUI +# Parameters of admin GUI +admin_gui = { + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } +} + +# Old GUI +admin_old_gui = { + api = { + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + old = { + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } +} + +# Parameters of the compute plane +compute_plane = { + # Default partition that uses the C# extension for the worker + default = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "test_image" + tag = "latest" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, + # Partition for the stream worker + stream = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "dockerhubaneo/armonik_core_stream_test_worker" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, + # Partition for the htcmock worker + htcmock = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "dockerhubaneo/armonik_core_htcmock_test_worker" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, + # Partition for the bench worker + bench = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "dockerhubaneo/armonik_core_bench_test_worker" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, +} + +# Deploy ingress +# PS: to not deploy ingress put: "ingress=null" +ingress = { + tls = false + mtls = false + generate_client_cert = false +} + +extra_conf = { + core = { + Amqp__AllowHostMismatch = true + Amqp__MaxPriority = "10" + Amqp__MaxRetries = "5" + Amqp__QueueStorage__LockRefreshPeriodicity = "00:00:45" + Amqp__QueueStorage__PollPeriodicity = "00:00:10" + Amqp__QueueStorage__LockRefreshExtension = "00:02:00" + MongoDB__TableStorage__PollingDelayMin = "00:00:01" + MongoDB__TableStorage__PollingDelayMax = "00:00:10" + MongoDB__AllowInsecureTls = true + MongoDB__TableStorage__PollingDelay = "00:00:01" + MongoDB__DataRetention = "10.00:00:00" + Redis__Timeout = 30000 + Redis__SslHost = "127.0.0.1" + } + control = { + Submitter__MaxErrorAllowed = 50 + } +} + +environment_description = { + name = "local-dev" + version = "0.0.0" + description = "Local development environment" + color = "blue" +}