diff --git a/libs/commons/src/main/kotlin/uk/gov/justice/digital/hmpps/retry/Retry.kt b/libs/commons/src/main/kotlin/uk/gov/justice/digital/hmpps/retry/Retry.kt index cf38551e1c..c3b9f565c2 100644 --- a/libs/commons/src/main/kotlin/uk/gov/justice/digital/hmpps/retry/Retry.kt +++ b/libs/commons/src/main/kotlin/uk/gov/justice/digital/hmpps/retry/Retry.kt @@ -1,13 +1,32 @@ package uk.gov.justice.digital.hmpps.retry -fun retry(maxRetries: Int, code: () -> T): T { - var throwable: Throwable? = null - (1..maxRetries).forEach { _ -> +import java.time.Duration +import java.util.concurrent.TimeUnit +import kotlin.reflect.KClass + +fun retry( + maxRetries: Int, + exceptions: List> = listOf(Exception::class), + delay: Duration = Duration.ofMillis(100), + code: () -> T +): T { + var throwable: Throwable? + (1..maxRetries).forEach { count -> try { return code() } catch (e: Throwable) { - throwable = e + val matchedException = exceptions.firstOrNull { it.isInstance(e) } + throwable = if (matchedException != null && count < maxRetries) { + null + } else { + e + } + if (throwable == null) { + TimeUnit.MILLISECONDS.sleep(delay.toMillis() * count * count) + } else { + throw throwable!! + } } } - throw throwable!! + throw RuntimeException("unknown error") } diff --git a/libs/commons/src/test/kotlin/uk/gov/justice/digital/hmpps/retry/RetryTest.kt b/libs/commons/src/test/kotlin/uk/gov/justice/digital/hmpps/retry/RetryTest.kt index ed0455696c..f5e86979ac 100644 --- a/libs/commons/src/test/kotlin/uk/gov/justice/digital/hmpps/retry/RetryTest.kt +++ b/libs/commons/src/test/kotlin/uk/gov/justice/digital/hmpps/retry/RetryTest.kt @@ -2,9 +2,12 @@ package uk.gov.justice.digital.hmpps.retry import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo +import org.json.JSONException import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.springframework.dao.OptimisticLockingFailureException import java.rmi.UnexpectedException +import java.sql.SQLException import java.util.concurrent.atomic.AtomicInteger internal class RetryTest { @@ -29,4 +32,28 @@ internal class RetryTest { } assertThat(result, equalTo(1)) } + + @Test + fun `when optimistic lock exception thrown retry until max retries`() { + val counter = AtomicInteger(0) + assertThrows { + retry(3, listOf(OptimisticLockingFailureException::class, JSONException::class)) { + counter.incrementAndGet() + throw OptimisticLockingFailureException("OLE") + } + } + assertThat(counter.get(), equalTo(3)) + } + + @Test + fun `when SQL exception thrown no retries`() { + val counter = AtomicInteger(0) + assertThrows { + retry(3, listOf(OptimisticLockingFailureException::class)) { + counter.incrementAndGet() + throw SQLException("SQLE") + } + } + assertThat(counter.get(), equalTo(1)) + } } diff --git a/libs/messaging/build.gradle.kts b/libs/messaging/build.gradle.kts index 5ec0ce6bb3..a92cd0a765 100644 --- a/libs/messaging/build.gradle.kts +++ b/libs/messaging/build.gradle.kts @@ -5,11 +5,15 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-web") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation(libs.bundles.telemetry) + compileOnly(libs.openfeign) + compileOnly("org.springframework.boot:spring-boot-starter-data-jpa") api(libs.bundles.aws.messaging) testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation(libs.bundles.mockito) + testImplementation(libs.openfeign) + testImplementation("org.springframework.boot:spring-boot-starter-data-jpa") } configure { diff --git a/libs/messaging/src/main/kotlin/uk/gov/justice/digital/hmpps/listener/AwsNotificationListener.kt b/libs/messaging/src/main/kotlin/uk/gov/justice/digital/hmpps/listener/AwsNotificationListener.kt index d1e739cd3b..65db7b458f 100644 --- a/libs/messaging/src/main/kotlin/uk/gov/justice/digital/hmpps/listener/AwsNotificationListener.kt +++ b/libs/messaging/src/main/kotlin/uk/gov/justice/digital/hmpps/listener/AwsNotificationListener.kt @@ -1,5 +1,6 @@ package uk.gov.justice.digital.hmpps.listener +import feign.FeignException import io.awspring.cloud.sqs.annotation.SqsListener import io.awspring.cloud.sqs.listener.AsyncAdapterBlockingExecutionFailedException import io.awspring.cloud.sqs.listener.ListenerExecutionFailedException @@ -9,9 +10,14 @@ import io.sentry.Sentry import io.sentry.spring.jakarta.tracing.SentryTransaction import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Conditional +import org.springframework.dao.CannotAcquireLockException +import org.springframework.jdbc.CannotGetJdbcConnectionException +import org.springframework.orm.ObjectOptimisticLockingFailureException import org.springframework.stereotype.Component +import org.springframework.transaction.CannotCreateTransactionException import uk.gov.justice.digital.hmpps.config.AwsCondition import uk.gov.justice.digital.hmpps.messaging.NotificationHandler +import uk.gov.justice.digital.hmpps.retry.retry import java.util.concurrent.CompletionException @Component @@ -25,7 +31,16 @@ class AwsNotificationListener( @WithSpan(kind = SpanKind.CONSUMER) fun receive(message: String) { try { - handler.handle(message) + retry( + 3, + listOf( + FeignException.NotFound::class, + CannotAcquireLockException::class, + ObjectOptimisticLockingFailureException::class, + CannotCreateTransactionException::class, + CannotGetJdbcConnectionException::class + ) + ) { handler.handle(message) } } catch (e: Throwable) { Sentry.captureException(unwrapSqsExceptions(e)) throw e diff --git a/libs/oauth-client/src/main/kotlin/uk/gov/justice/digital/hmpps/config/feign/FeignConfig.kt b/libs/oauth-client/src/main/kotlin/uk/gov/justice/digital/hmpps/config/feign/FeignConfig.kt index 640cc2548c..62ea960dd1 100644 --- a/libs/oauth-client/src/main/kotlin/uk/gov/justice/digital/hmpps/config/feign/FeignConfig.kt +++ b/libs/oauth-client/src/main/kotlin/uk/gov/justice/digital/hmpps/config/feign/FeignConfig.kt @@ -1,6 +1,7 @@ package uk.gov.justice.digital.hmpps.config.feign import feign.RequestInterceptor +import feign.Retryer import org.springframework.context.annotation.Bean import org.springframework.http.HttpHeaders import org.springframework.security.authentication.AnonymousAuthenticationToken @@ -16,6 +17,9 @@ abstract class FeignConfig( abstract fun registrationId(): String + @Bean + open fun retryer() = Retryer.Default() + @Bean open fun requestInterceptor() = RequestInterceptor { template -> template.header(HttpHeaders.AUTHORIZATION, "Bearer ${getAccessToken()}")