PI-1478 changes to retry exceptions (#2300)

libs/commons/src/main/kotlin/uk/gov/justice/digital/hmpps/retry/Retry.kt

@@ -1,13 +1,32 @@
 package uk.gov.justice.digital.hmpps.retry
 
-fun <T> retry(maxRetries: Int, code: () -> T): T {
-    var throwable: Throwable? = null
-    (1..maxRetries).forEach { _ ->
+import java.time.Duration
+import java.util.concurrent.TimeUnit
+import kotlin.reflect.KClass
+
+fun <T> retry(
+    maxRetries: Int,
+    exceptions: List<KClass<out Exception>> = listOf(Exception::class),
+    delay: Duration = Duration.ofMillis(100),
+    code: () -> T
+): T {
+    var throwable: Throwable?
+    (1..maxRetries).forEach { count ->
         try {
             return code()
         } catch (e: Throwable) {
-            throwable = e
+            val matchedException = exceptions.firstOrNull { it.isInstance(e) }
+            throwable = if (matchedException != null && count < maxRetries) {
+                null
+            } else {
+                e
+            }
+            if (throwable == null) {
+                TimeUnit.MILLISECONDS.sleep(delay.toMillis() * count * count)
+            } else {
+                throw throwable!!
+            }
         }
     }
-    throw throwable!!
+    throw RuntimeException("unknown error")
 }

libs/commons/src/test/kotlin/uk/gov/justice/digital/hmpps/retry/RetryTest.kt

@@ -2,9 +2,12 @@ package uk.gov.justice.digital.hmpps.retry
 
 import org.hamcrest.MatcherAssert.assertThat
 import org.hamcrest.Matchers.equalTo
+import org.json.JSONException
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.springframework.dao.OptimisticLockingFailureException
 import java.rmi.UnexpectedException
+import java.sql.SQLException
 import java.util.concurrent.atomic.AtomicInteger
 
 internal class RetryTest {
@@ -29,4 +32,28 @@ internal class RetryTest {
         }
         assertThat(result, equalTo(1))
     }
+
+    @Test
+    fun `when optimistic lock exception thrown retry until max retries`() {
+        val counter = AtomicInteger(0)
+        assertThrows<OptimisticLockingFailureException> {
+            retry(3, listOf(OptimisticLockingFailureException::class, JSONException::class)) {
+                counter.incrementAndGet()
+                throw OptimisticLockingFailureException("OLE")
+            }
+        }
+        assertThat(counter.get(), equalTo(3))
+    }
+
+    @Test
+    fun `when SQL exception thrown no retries`() {
+        val counter = AtomicInteger(0)
+        assertThrows<SQLException> {
+            retry(3, listOf(OptimisticLockingFailureException::class)) {
+                counter.incrementAndGet()
+                throw SQLException("SQLE")
+            }
+        }
+        assertThat(counter.get(), equalTo(1))
+    }
 }

libs/messaging/build.gradle.kts

@@ -5,11 +5,15 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-web")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
     implementation(libs.bundles.telemetry)
+    compileOnly(libs.openfeign)
+    compileOnly("org.springframework.boot:spring-boot-starter-data-jpa")
 
     api(libs.bundles.aws.messaging)
 
     testImplementation("org.springframework.boot:spring-boot-starter-test")
     testImplementation(libs.bundles.mockito)
+    testImplementation(libs.openfeign)
+    testImplementation("org.springframework.boot:spring-boot-starter-data-jpa")
 }
 
 configure<ClassPathExtension> {

libs/messaging/src/main/kotlin/uk/gov/justice/digital/hmpps/listener/AwsNotificationListener.kt

@@ -1,5 +1,6 @@
 package uk.gov.justice.digital.hmpps.listener
 
+import feign.FeignException
 import io.awspring.cloud.sqs.annotation.SqsListener
 import io.awspring.cloud.sqs.listener.AsyncAdapterBlockingExecutionFailedException
 import io.awspring.cloud.sqs.listener.ListenerExecutionFailedException
@@ -9,9 +10,14 @@ import io.sentry.Sentry
 import io.sentry.spring.jakarta.tracing.SentryTransaction
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression
 import org.springframework.context.annotation.Conditional
+import org.springframework.dao.CannotAcquireLockException
+import org.springframework.jdbc.CannotGetJdbcConnectionException
+import org.springframework.orm.ObjectOptimisticLockingFailureException
 import org.springframework.stereotype.Component
+import org.springframework.transaction.CannotCreateTransactionException
 import uk.gov.justice.digital.hmpps.config.AwsCondition
 import uk.gov.justice.digital.hmpps.messaging.NotificationHandler
+import uk.gov.justice.digital.hmpps.retry.retry
 import java.util.concurrent.CompletionException
 
 @Component
@@ -25,7 +31,16 @@ class AwsNotificationListener(
     @WithSpan(kind = SpanKind.CONSUMER)
     fun receive(message: String) {
         try {
-            handler.handle(message)
+            retry(
+                3,
+                listOf(
+                    FeignException.NotFound::class,
+                    CannotAcquireLockException::class,
+                    ObjectOptimisticLockingFailureException::class,
+                    CannotCreateTransactionException::class,
+                    CannotGetJdbcConnectionException::class
+                )
+            ) { handler.handle(message) }
         } catch (e: Throwable) {
             Sentry.captureException(unwrapSqsExceptions(e))
             throw e

libs/oauth-client/src/main/kotlin/uk/gov/justice/digital/hmpps/config/feign/FeignConfig.kt

@@ -1,6 +1,7 @@
 package uk.gov.justice.digital.hmpps.config.feign
 
 import feign.RequestInterceptor
+import feign.Retryer
 import org.springframework.context.annotation.Bean
 import org.springframework.http.HttpHeaders
 import org.springframework.security.authentication.AnonymousAuthenticationToken
@@ -16,6 +17,9 @@ abstract class FeignConfig(
 
     abstract fun registrationId(): String
 
+    @Bean
+    open fun retryer() = Retryer.Default()
+
     @Bean
     open fun requestInterceptor() = RequestInterceptor { template ->
         template.header(HttpHeaders.AUTHORIZATION, "Bearer ${getAccessToken()}")