From ad76effeef742e2677f37ad4823b34e9602d54b5 Mon Sep 17 00:00:00 2001 From: Dylan Brasseur Date: Mon, 16 Oct 2023 10:23:05 +0200 Subject: [PATCH 1/5] Fix retry delay --- Client/src/Common/Properties.cs | 70 ++++++++++++++++--- .../src/Unified/Services/Submitter/Service.cs | 8 ++- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/Client/src/Common/Properties.cs b/Client/src/Common/Properties.cs index 1aec5ba0..132452f9 100644 --- a/Client/src/Common/Properties.cs +++ b/Client/src/Common/Properties.cs @@ -54,6 +54,10 @@ public class Properties private const string SectionClientCertP12 = "ClientP12"; private const string SectionTargetNameOverride = "EndpointNameOverride"; + private const string SectionRetryInitialBackoff = "RetryInitialBackoff"; + private const string SectionRetryBackoffMultiplier = "RetryBackoffMultiplier"; + private const string SectionRetryMaxBackoff = "RetryMaxBackoff"; + /// /// The default configuration to submit task in a Session /// @@ -116,17 +120,23 @@ public Properties(TaskOptions options, /// The client key file in a pem format /// The client certificate in a P12/Pkcs12/PFX format /// Disable the ssl strong validation of ssl certificate (default : enable => true) + /// Initial retry backoff delay + /// Retry backoff multiplier + /// Max retry backoff /// public Properties(IConfiguration configuration, TaskOptions options, - string connectionAddress = null, - int connectionPort = 0, - string protocol = null, - string clientCertFilePem = null, - string clientKeyFilePem = null, - string clientP12 = null, - string caCertPem = null, - bool? sslValidation = null) + string connectionAddress = null, + int connectionPort = 0, + string protocol = null, + string clientCertFilePem = null, + string clientKeyFilePem = null, + string clientP12 = null, + string caCertPem = null, + bool? sslValidation = null, + TimeSpan retryInitialBackoff = new(), + double retryBackoffMultiplier = 0, + TimeSpan retryMaxBackoff = new()) { TaskOptions = options; Configuration = configuration; @@ -160,6 +170,35 @@ public Properties(IConfiguration configuration, ClientKeyFilePem = clientKeyFilePem ?? sectionGrpc?[SectionClientKey]; ClientP12File = clientP12 ?? sectionGrpc?[SectionClientCertP12]; + if (retryInitialBackoff != TimeSpan.Zero) + { + RetryInitialBackoff = retryInitialBackoff; + } + else if (!string.IsNullOrWhiteSpace(sectionGrpc?[SectionRetryInitialBackoff])) + { + RetryInitialBackoff = TimeSpan.Parse(sectionGrpc[SectionRetryInitialBackoff]); + } + + if (retryBackoffMultiplier != 0) + { + RetryBackoffMultiplier = retryBackoffMultiplier; + } + else if (!string.IsNullOrWhiteSpace(sectionGrpc?[SectionRetryBackoffMultiplier])) + { + RetryBackoffMultiplier = double.Parse(sectionGrpc[SectionRetryBackoffMultiplier]); + } + + + if (retryMaxBackoff != TimeSpan.Zero) + { + RetryMaxBackoff = retryMaxBackoff; + } + else if (!string.IsNullOrWhiteSpace(sectionGrpc?[SectionRetryMaxBackoff])) + { + RetryMaxBackoff = TimeSpan.Parse(sectionGrpc[SectionRetryMaxBackoff]); + } + + if (connectionPort != 0) { ConnectionPort = connectionPort; @@ -285,4 +324,19 @@ public string ConnectionString /// The target name of the endpoint when ssl validation is disabled. Automatic if not set. /// public string TargetNameOverride { get; set; } = ""; + + /// + /// Initial backoff from retries + /// + public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Backoff multiplier for retries + /// + public double RetryBackoffMultiplier { get; set; } = 2; + + /// + /// Max backoff for retries + /// + public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromSeconds(30); } diff --git a/Client/src/Unified/Services/Submitter/Service.cs b/Client/src/Unified/Services/Submitter/Service.cs index 35a76f57..c9668a3f 100644 --- a/Client/src/Unified/Services/Submitter/Service.cs +++ b/Client/src/Unified/Services/Submitter/Service.cs @@ -152,6 +152,7 @@ public Service(Properties properties, { var maxRetries = groupBlockRequest.First() .MaxRetries; + var currentBackoff = properties.RetryInitialBackoff; for (var retry = 0; retry < maxRetries; retry++) { //Generate resultId @@ -214,11 +215,14 @@ public Service(Properties properties, Logger?.LogWarning(e, "Fail to submit, {retry}/{maxRetries} retrying", - retry, + retry + 1, maxRetries); //Delay before submission - Task.Delay(TimeSpan.FromMilliseconds(1000)); + Task.Delay(currentBackoff) + .Wait(); + currentBackoff = TimeSpan.FromSeconds(Math.Min(currentBackoff.TotalSeconds * properties.RetryBackoffMultiplier, + properties.RetryMaxBackoff.TotalSeconds)); } } From 7f4d6c1c6257ac8d29fcd6babe821245bcc83529 Mon Sep 17 00:00:00 2001 From: Dylan Brasseur Date: Mon, 16 Oct 2023 11:52:03 +0200 Subject: [PATCH 2/5] Fixed build core version --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be693781..61115bec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -213,6 +213,7 @@ jobs: tls: ${{ matrix.tls }} mtls: ${{ matrix.mtls }} ext-csharp-version: ${{ needs.versionning.outputs.version }} + core-version: 0.13.2 - name: Setup hosts file run : echo -e "$(kubectl get svc ingress -n armonik -o jsonpath={.status.loadBalancer.ingress[0].ip})\tarmonik.local" | sudo tee -a /etc/hosts From 5697121e5330b1b66caddbdec6b6a57ab5aad456 Mon Sep 17 00:00:00 2001 From: Dylan Brasseur Date: Tue, 20 Jun 2023 11:00:43 +0200 Subject: [PATCH 3/5] Fixes and reduces nuint tests --- .../Tests/AggregationPriority/AggregationPriorityTest.cs | 1 + .../ArmoniK.EndToEndTests.Client/Tests/UnitTestHelperBase.cs | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs index eedad9fc..7ccf3a6f 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs @@ -303,6 +303,7 @@ private IEnumerable> WaitForResults(string se /// /// The size of the square matrix. [TestCase(20)] + [Ignore("Too big")] public void Check_That_Result_has_expected_value(int squareMatrixSize) { unifiedTestHelper_.Log.LogInformation($"Compute square matrix with n = {squareMatrixSize}"); diff --git a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/UnitTestHelperBase.cs b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/UnitTestHelperBase.cs index 027a8b5d..ea19a6c9 100644 --- a/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/UnitTestHelperBase.cs +++ b/Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/UnitTestHelperBase.cs @@ -85,9 +85,8 @@ public void InitProperties(EngineType engineType, applicationNamespace, applicationService); - Props = new Properties(TaskOptions, - Configuration.GetSection("Grpc")["EndPoint"], - 5001); + Props = new Properties(Configuration, + TaskOptions); } public static object[] ParamsHelper(params object[] elements) From e58e3c7942f6e84504e6192a419db09fdb5e7a06 Mon Sep 17 00:00:00 2001 From: Dylan Brasseur Date: Mon, 16 Oct 2023 14:17:34 +0200 Subject: [PATCH 4/5] Only get for properties --- Client/src/Common/Properties.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Client/src/Common/Properties.cs b/Client/src/Common/Properties.cs index 132452f9..eaee8055 100644 --- a/Client/src/Common/Properties.cs +++ b/Client/src/Common/Properties.cs @@ -328,15 +328,15 @@ public string ConnectionString /// /// Initial backoff from retries /// - public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(1); + public TimeSpan RetryInitialBackoff { get; } = TimeSpan.FromSeconds(1); /// /// Backoff multiplier for retries /// - public double RetryBackoffMultiplier { get; set; } = 2; + public double RetryBackoffMultiplier { get; } = 2; /// /// Max backoff for retries /// - public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromSeconds(30); + public TimeSpan RetryMaxBackoff { get; } = TimeSpan.FromSeconds(30); } From 60d1a29dfa75c6d07c048e03385e75b6441bf732 Mon Sep 17 00:00:00 2001 From: Dylan Brasseur Date: Mon, 16 Oct 2023 14:47:05 +0200 Subject: [PATCH 5/5] Removed ci destroy deployment --- .github/workflows/build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61115bec..ad25bd12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -261,13 +261,6 @@ jobs: name: "IntegrationTests tls:${{ matrix.tls }} mtls:${{ matrix.mtls }} val:${{ matrix.sslvalidation }} ca:${{ matrix.useca }}" path: ./Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/TestResults/test-results.trx reporter: dotnet-trx - - - name: Destroy deployment - if: success() || failure() - uses: aneoconsulting/ArmoniK.Action.Deploy/destroy@main - with: - working-directory: ${{ github.workspace }}/infra - type: localhost canMerge: needs: