chore(sql): jitter around sql write retries (#63)

spinnaker · Sep 13, 2019 · ff26d8f · ff26d8f
1 parent 3101c2d
commit ff26d8f
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/keiko-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlQueueProperties.kt b/keiko-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlQueueProperties.kt
@@ -52,7 +52,7 @@ class SqlQueueProperties {
    * See: https://github.com/spinnaker/kork/blob/master/kork-sql/src/main/kotlin/com/netflix/spinnaker/kork/sql/config/SqlRetryProperties.kt
    */
   var retries: SqlRetryProperties = SqlRetryProperties(
-    transactions = RetryProperties(maxRetries = 10, backoffMs = 100),
+    transactions = RetryProperties(maxRetries = 20, backoffMs = 100),
     reads = RetryProperties()
   )
 }
diff --git a/keiko-sql/src/main/kotlin/com/netflix/spinnaker/q/sql/SqlQueue.kt b/keiko-sql/src/main/kotlin/com/netflix/spinnaker/q/sql/SqlQueue.kt
@@ -55,6 +55,7 @@ import java.util.concurrent.TimeUnit
 import kotlin.Exception
 import kotlin.math.max
 import kotlin.math.min
+import kotlin.random.Random.Default.nextLong
 
 @KotlinOpen
 class SqlQueue(
@@ -116,6 +117,9 @@ class SqlQueue(
 
   private val lockTtlDuration = Duration.ofSeconds(lockTtlSeconds.toLong())
 
+  private val writeRetryBackoffMin = max(sqlRetryProperties.transactions.backoffMs - 25, 25)
+  private val writeRetryBackoffMax = max(sqlRetryProperties.transactions.backoffMs + 50, 100)
+
   init {
     log.info("Configured $javaClass queue: $queueName")
     initTables()
@@ -716,7 +720,9 @@ class SqlQueue(
         "sqlWrite",
         RetryConfig.custom<T>()
           .maxAttempts(sqlRetryProperties.transactions.maxRetries)
-          .waitDuration(Duration.ofMillis(sqlRetryProperties.transactions.backoffMs))
+          .waitDuration(
+            Duration.ofMillis(
+              nextLong(writeRetryBackoffMin, writeRetryBackoffMax)))
           .ignoreExceptions(SQLDialectNotSupportedException::class.java)
           .build()
       )