fix: Fix retry delay (#219)

.github/workflows/build.yml

@@ -213,6 +213,7 @@ jobs:
           tls: ${{ matrix.tls }}
           mtls: ${{ matrix.mtls }}
           ext-csharp-version: ${{ needs.versionning.outputs.version }}
+          core-version: 0.13.2
 
       - name: Setup hosts file
         run : echo -e "$(kubectl get svc ingress -n armonik -o jsonpath={.status.loadBalancer.ingress[0].ip})\tarmonik.local" | sudo tee -a /etc/hosts
@@ -260,13 +261,6 @@ jobs:
           name: "IntegrationTests tls:${{ matrix.tls }} mtls:${{ matrix.mtls }} val:${{ matrix.sslvalidation }} ca:${{ matrix.useca }}"
           path: ./Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/TestResults/test-results.trx
           reporter: dotnet-trx
-
-      - name: Destroy deployment
-        if: success() || failure()
-        uses: aneoconsulting/ArmoniK.Action.Deploy/destroy@main
-        with:
-          working-directory: ${{ github.workspace }}/infra
-          type: localhost
 
   canMerge:
     needs: 

Client/src/Common/Properties.cs

@@ -54,6 +54,10 @@ public class Properties
   private const string SectionClientCertP12      = "ClientP12";
   private const string SectionTargetNameOverride = "EndpointNameOverride";
 
+  private const string SectionRetryInitialBackoff    = "RetryInitialBackoff";
+  private const string SectionRetryBackoffMultiplier = "RetryBackoffMultiplier";
+  private const string SectionRetryMaxBackoff        = "RetryMaxBackoff";
+
   /// <summary>
   ///   The default configuration to submit task in a Session
   /// </summary>
@@ -116,17 +120,23 @@ public Properties(TaskOptions options,
   /// <param name="clientKeyFilePem">The client key file in a pem format</param>
   /// <param name="clientP12">The client certificate in a P12/Pkcs12/PFX format</param>
   /// <param name="sslValidation">Disable the ssl strong validation of ssl certificate (default : enable => true)</param>
+  /// <param name="retryInitialBackoff">Initial retry backoff delay</param>
+  /// <param name="retryBackoffMultiplier">Retry backoff multiplier</param>
+  /// <param name="retryMaxBackoff">Max retry backoff</param>
   /// <exception cref="ArgumentException"></exception>
   public Properties(IConfiguration configuration,
                     TaskOptions    options,
-                    string         connectionAddress = null,
-                    int            connectionPort    = 0,
-                    string         protocol          = null,
-                    string         clientCertFilePem = null,
-                    string         clientKeyFilePem  = null,
-                    string         clientP12         = null,
-                    string         caCertPem         = null,
-                    bool?          sslValidation     = null)
+                    string         connectionAddress      = null,
+                    int            connectionPort         = 0,
+                    string         protocol               = null,
+                    string         clientCertFilePem      = null,
+                    string         clientKeyFilePem       = null,
+                    string         clientP12              = null,
+                    string         caCertPem              = null,
+                    bool?          sslValidation          = null,
+                    TimeSpan       retryInitialBackoff    = new(),
+                    double         retryBackoffMultiplier = 0,
+                    TimeSpan       retryMaxBackoff        = new())
   {
     TaskOptions   = options;
     Configuration = configuration;
@@ -160,6 +170,35 @@ public Properties(IConfiguration configuration,
     ClientKeyFilePem   = clientKeyFilePem  ?? sectionGrpc?[SectionClientKey];
     ClientP12File      = clientP12         ?? sectionGrpc?[SectionClientCertP12];
 
+    if (retryInitialBackoff != TimeSpan.Zero)
+    {
+      RetryInitialBackoff = retryInitialBackoff;
+    }
+    else if (!string.IsNullOrWhiteSpace(sectionGrpc?[SectionRetryInitialBackoff]))
+    {
+      RetryInitialBackoff = TimeSpan.Parse(sectionGrpc[SectionRetryInitialBackoff]);
+    }
+
+    if (retryBackoffMultiplier != 0)
+    {
+      RetryBackoffMultiplier = retryBackoffMultiplier;
+    }
+    else if (!string.IsNullOrWhiteSpace(sectionGrpc?[SectionRetryBackoffMultiplier]))
+    {
+      RetryBackoffMultiplier = double.Parse(sectionGrpc[SectionRetryBackoffMultiplier]);
+    }
+
+
+    if (retryMaxBackoff != TimeSpan.Zero)
+    {
+      RetryMaxBackoff = retryMaxBackoff;
+    }
+    else if (!string.IsNullOrWhiteSpace(sectionGrpc?[SectionRetryMaxBackoff]))
+    {
+      RetryMaxBackoff = TimeSpan.Parse(sectionGrpc[SectionRetryMaxBackoff]);
+    }
+
+
     if (connectionPort != 0)
     {
       ConnectionPort = connectionPort;
@@ -285,4 +324,19 @@ public string ConnectionString
   ///   The target name of the endpoint when ssl validation is disabled. Automatic if not set.
   /// </summary>
   public string TargetNameOverride { get; set; } = "";
+
+  /// <summary>
+  ///   Initial backoff from retries
+  /// </summary>
+  public TimeSpan RetryInitialBackoff { get; } = TimeSpan.FromSeconds(1);
+
+  /// <summary>
+  ///   Backoff multiplier for retries
+  /// </summary>
+  public double RetryBackoffMultiplier { get; } = 2;
+
+  /// <summary>
+  ///   Max backoff for retries
+  /// </summary>
+  public TimeSpan RetryMaxBackoff { get; } = TimeSpan.FromSeconds(30);
 }

Client/src/Unified/Services/Submitter/Service.cs

@@ -152,6 +152,7 @@ public Service(Properties                 properties,
                                   {
                                     var maxRetries = groupBlockRequest.First()
                                                                       .MaxRetries;
+                                    var currentBackoff = properties.RetryInitialBackoff;
                                     for (var retry = 0; retry < maxRetries; retry++)
                                     {
                                       //Generate resultId
@@ -214,11 +215,14 @@ public Service(Properties                 properties,
 
                                         Logger?.LogWarning(e,
                                                            "Fail to submit, {retry}/{maxRetries} retrying",
-                                                           retry,
+                                                           retry + 1,
                                                            maxRetries);
 
                                         //Delay before submission
-                                        Task.Delay(TimeSpan.FromMilliseconds(1000));
+                                        Task.Delay(currentBackoff)
+                                            .Wait();
+                                        currentBackoff = TimeSpan.FromSeconds(Math.Min(currentBackoff.TotalSeconds * properties.RetryBackoffMultiplier,
+                                                                                       properties.RetryMaxBackoff.TotalSeconds));
                                       }
                                     }
 

Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/AggregationPriority/AggregationPriorityTest.cs

@@ -303,6 +303,7 @@ private IEnumerable<Tuple<string, byte[]>> WaitForResults(string              se
   /// </summary>
   /// <param name="squareMatrixSize">The size of the square matrix.</param>
   [TestCase(20)]
+  [Ignore("Too big")]
   public void Check_That_Result_has_expected_value(int squareMatrixSize)
   {
     unifiedTestHelper_.Log.LogInformation($"Compute square matrix with n =  {squareMatrixSize}");

Tests/ArmoniK.EndToEndTests/ArmoniK.EndToEndTests.Client/Tests/UnitTestHelperBase.cs

@@ -85,9 +85,8 @@ public void InitProperties(EngineType engineType,
                                         applicationNamespace,
                                         applicationService);
 
-    Props = new Properties(TaskOptions,
-                           Configuration.GetSection("Grpc")["EndPoint"],
-                           5001);
+    Props = new Properties(Configuration,
+                           TaskOptions);
   }
 
   public static object[] ParamsHelper(params object[] elements)