From dd883c6a24f883ccd410ac897f2e16cb3c84e266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 24 Jul 2023 14:24:26 +0200 Subject: [PATCH 01/18] feature/Add endpoint getEntitlementsAndPermissions v5.1.0 --- .../scala/code/api/v5_1_0/APIMethods510.scala | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala index e436c1cd81..baeb752427 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala @@ -8,6 +8,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFound, InvalidJsonFormat, UnknownError, UserNotFoundByUserId, UserNotLoggedIn, _} import code.api.util.NewStyle.HttpCode import code.api.util._ +import code.api.v2_0_0.{EntitlementJSONs, JSONFactory200} import code.api.v3_0_0.JSONFactory300 import code.api.v3_0_0.JSONFactory300.createAggregateMetricJson import code.api.v3_1_0.ConsentJsonV310 @@ -24,12 +25,13 @@ import code.transactionrequests.TransactionRequests.TransactionRequestTypes.{app import code.userlocks.UserLocksProvider import code.users.Users import code.util.Helper +import code.views.Views import code.views.system.{AccountAccess, ViewDefinition} import com.github.dwickern.macros.NameOf.nameOf import com.openbankproject.commons.ExecutionContext.Implicits.global import com.openbankproject.commons.dto.CustomerAndAttribute import com.openbankproject.commons.model.enums.{AtmAttributeType, UserAttributeType} -import com.openbankproject.commons.model.{AtmId, AtmT, BankId} +import com.openbankproject.commons.model.{AtmId, AtmT, BankId, Permission} import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion} import net.liftweb.common.{Box, Full} import net.liftweb.http.S @@ -285,6 +287,43 @@ trait APIMethods510 { } } } + + + + staticResourceDocs += ResourceDoc( + getEntitlementsAndPermissions, + implementedInApiVersion, + "getEntitlementsAndPermissions", + "GET", + "/users/USER_ID/entitlements", + "Get Entitlements and Permissions for a User", + s""" + | + | + """.stripMargin, + EmptyBody, + userJsonV300, + List( + $UserNotLoggedIn, + UserNotFoundByUserId, + UserHasMissingRoles, + UnknownError), + List(apiTagRole, apiTagEntitlement, apiTagUser), + Some(List(canGetEntitlementsForAnyUserAtAnyBank))) + + + lazy val getEntitlementsAndPermissions: OBPEndpoint = { + case "users" :: userId :: "entitlements-and-permissions" :: Nil JsonGet _ => { + cc => + for { + (user, callContext) <- NewStyle.function.getUserByUserId(userId, cc.callContext) + entitlements <- NewStyle.function.getEntitlementsByUserId(userId, callContext) + } yield { + val permissions: Option[Permission] = Views.views.vend.getPermissionForUser(user).toOption + (JSONFactory300.createUserInfoJSON (user, entitlements, permissions), HttpCode.`200`(callContext)) + } + } + } staticResourceDocs += ResourceDoc( From 0721d45e95a764e5b7eced0aeea45705205439bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 25 Jul 2023 14:57:30 +0200 Subject: [PATCH 02/18] test/Add tests for endpoint getEntitlementsAndPermissions v5.1.0 --- .../scala/code/api/v5_1_0/APIMethods510.scala | 2 +- .../test/scala/code/api/v5_1_0/UserTest.scala | 52 +++++++++++++++++-- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala index baeb752427..2fa5abe4a9 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala @@ -295,7 +295,7 @@ trait APIMethods510 { implementedInApiVersion, "getEntitlementsAndPermissions", "GET", - "/users/USER_ID/entitlements", + "/users/USER_ID/entitlements-and-permissions", "Get Entitlements and Permissions for a User", s""" | diff --git a/obp-api/src/test/scala/code/api/v5_1_0/UserTest.scala b/obp-api/src/test/scala/code/api/v5_1_0/UserTest.scala index 6a9d1c702b..44c380ac02 100644 --- a/obp-api/src/test/scala/code/api/v5_1_0/UserTest.scala +++ b/obp-api/src/test/scala/code/api/v5_1_0/UserTest.scala @@ -1,10 +1,12 @@ package code.api.v5_1_0 +import java.util.UUID + import code.api.util.APIUtil.OAuth._ -import code.api.util.ApiRole.CanGetAnyUser +import code.api.util.ApiRole.{CanGetAnyUser, CanGetEntitlementsForAnyUserAtAnyBank} import code.api.util.ErrorMessages.{UserHasMissingRoles, UserNotLoggedIn, attemptedToOpenAnEmptyBox} -import code.api.v4_0_0.OBPAPI4_0_0.Implementations4_0_0 -import code.api.v4_0_0.{UserIdJsonV400, UserJsonV400} +import code.api.v3_0_0.UserJsonV300 +import code.api.v4_0_0.UserJsonV400 import code.api.v5_1_0.OBPAPI5_1_0.Implementations5_1_0 import code.entitlement.Entitlement import code.model.UserX @@ -14,8 +16,6 @@ import com.openbankproject.commons.model.ErrorMessage import com.openbankproject.commons.util.ApiVersion import org.scalatest.Tag -import java.util.UUID - class UserTest extends V510ServerSetup { /** * Test tags @@ -26,6 +26,7 @@ class UserTest extends V510ServerSetup { */ object VersionOfApi extends Tag(ApiVersion.v5_1_0.toString) object ApiEndpoint1 extends Tag(nameOf(Implementations5_1_0.getUserByProviderAndUsername)) + object ApiEndpoint2 extends Tag(nameOf(Implementations5_1_0.getEntitlementsAndPermissions)) feature(s"test $ApiEndpoint1 version $VersionOfApi - Unauthorized access") { scenario("We will call the endpoint without user credentials", ApiEndpoint1, VersionOfApi) { @@ -62,5 +63,46 @@ class UserTest extends V510ServerSetup { Users.users.vend.deleteResourceUser(user.id.get) } } + + + + feature(s"test $ApiEndpoint2 version $VersionOfApi - Unauthorized access") { + scenario("We will call the endpoint without user credentials", ApiEndpoint1, VersionOfApi) { + When("We make a request v5.1.0") + val request = (v5_1_0_Request / "users" / "USER_ID" / "entitlements-and-permissions").GET + val response = makeGetRequest(request) + Then("We should get a 401") + response.code should equal(401) + response.body.extract[ErrorMessage].message should equal(UserNotLoggedIn) + } + } + feature(s"test $ApiEndpoint2 version $VersionOfApi - Authorized access") { + scenario("We will call the endpoint with user credentials but without a proper entitlement", ApiEndpoint1, VersionOfApi) { + val user = UserX.createResourceUser(defaultProvider, Some("user.name.1"), None, Some("user.name.1"), None, Some(UUID.randomUUID.toString), None).openOrThrowException(attemptedToOpenAnEmptyBox) + When("We make a request v5.1.0") + val request = (v5_1_0_Request / "users" / user.userId / "entitlements-and-permissions").GET <@(user1) + val response = makeGetRequest(request) + Then("error should be " + UserHasMissingRoles + CanGetEntitlementsForAnyUserAtAnyBank) + response.code should equal(403) + response.body.extract[ErrorMessage].message should be (UserHasMissingRoles + CanGetEntitlementsForAnyUserAtAnyBank) + // Clean up + Users.users.vend.deleteResourceUser(user.id.get) + } + } + feature(s"test $ApiEndpoint2 version $VersionOfApi - Authorized access") { + scenario("We will call the endpoint with user credentials and a proper entitlement", ApiEndpoint1, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanGetEntitlementsForAnyUserAtAnyBank.toString) + val user = UserX.createResourceUser(defaultProvider, Some("user.name.1"), None, Some("user.name.1"), None, Some(UUID.randomUUID.toString), None).openOrThrowException(attemptedToOpenAnEmptyBox) + When("We make a request v5.1.0") + val request = (v5_1_0_Request / "users" / user.userId / "entitlements-and-permissions").GET <@(user1) + val response = makeGetRequest(request) + Then("We get successful response") + response.code should equal(200) + response.body.extract[UserJsonV300] + // Clean up + Users.users.vend.deleteResourceUser(user.id.get) + } + } + } From 3efe58224e1247b388621a6c5df19216531e82c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 1 Aug 2023 15:26:53 +0200 Subject: [PATCH 03/18] docfix/Log DB connection warnings --- .../scheduler/DatabaseDriverScheduler.scala | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/scheduler/DatabaseDriverScheduler.scala b/obp-api/src/main/scala/code/scheduler/DatabaseDriverScheduler.scala index 3be284c9c0..c31fe50868 100644 --- a/obp-api/src/main/scala/code/scheduler/DatabaseDriverScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/DatabaseDriverScheduler.scala @@ -5,7 +5,7 @@ import java.util.concurrent.TimeUnit import code.actorsystem.ObpLookupSystem import code.util.Helper.MdcLoggable -import net.liftweb.db.DB +import net.liftweb.db.{DB, SuperConnection} import scala.concurrent.duration._ @@ -25,11 +25,26 @@ object DatabaseDriverScheduler extends MdcLoggable { } ) } + + def logWarnings(conn: SuperConnection) = { + var warning = conn.getWarnings() + if (warning != null) { + logger.warn("---Warning---") + while (warning != null) + { + logger.warn("Message: " + warning.getMessage()) + logger.warn("SQLState: " + warning.getSQLState()) + logger.warn("Vendor error code: " + warning.getErrorCode()) + warning = warning.getNextWarning() + } + } + } def clearAllMessages() = { DB.use(net.liftweb.util.DefaultConnectionIdentifier) { conn => try { + logWarnings(conn) conn.clearWarnings() logger.warn("DatabaseDriverScheduler.clearAllMessages - DONE") } catch { From 3494ca5d2fd7807f90727965e3378082ca845300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 2 Aug 2023 14:04:23 +0200 Subject: [PATCH 04/18] bugfix/Fix error handling during creating Hydra clients at boot time --- .../main/scala/bootstrap/liftweb/Boot.scala | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 941bac83b4..d92a482ec5 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -137,7 +137,7 @@ import com.openbankproject.commons.util.Functions.Implicits._ import com.openbankproject.commons.util.{ApiVersion, Functions} import javax.mail.internet.MimeMessage import net.liftweb.common._ -import net.liftweb.db.DBLogEntry +import net.liftweb.db.{DB, DBLogEntry} import net.liftweb.http.LiftRules.DispatchPF import net.liftweb.http._ import net.liftweb.http.provider.HTTPCookie @@ -835,15 +835,26 @@ class Boot extends MdcLoggable { // create Hydra client if exists active consumer but missing Hydra client def createHydraClients() = { - import scala.concurrent.ExecutionContext.Implicits.global - // exists hydra clients id - val oAuth2ClientIds = HydraUtil.hydraAdmin.listOAuth2Clients(Long.MaxValue, 0L).stream() - .map[String](_.getClientId) - .collect(Collectors.toSet()) - - Consumers.consumers.vend.getConsumersFuture().foreach{ consumers => - consumers.filter(consumer => consumer.isActive.get && !oAuth2ClientIds.contains(consumer.key.get)) - .foreach(HydraUtil.createHydraClient(_)) + try { + import scala.concurrent.ExecutionContext.Implicits.global + // exists hydra clients id + val oAuth2ClientIds = HydraUtil.hydraAdmin.listOAuth2Clients(Long.MaxValue, 0L).stream() + .map[String](_.getClientId) + .collect(Collectors.toSet()) + + Consumers.consumers.vend.getConsumersFuture().foreach{ consumers => + consumers.filter(consumer => consumer.isActive.get && !oAuth2ClientIds.contains(consumer.key.get)) + .foreach(HydraUtil.createHydraClient(_)) + } + } catch { + case e: Exception => + if(HydraUtil.integrateWithHydra) { + logger.error("------------------------------ Mirror consumer in hydra issue ------------------------------") + e.printStackTrace() + } else { + logger.warn("------------------------------ Mirror consumer in hydra issue ------------------------------") + logger.warn(e) + } } } From 00d435132fb5713fa61c2349d70651c53ff3ee8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Thu, 3 Aug 2023 11:08:57 +0200 Subject: [PATCH 05/18] refactor/Remove scalikejdbc in case of get top APIs --- .../scala/code/metrics/MappedMetrics.scala | 85 ++++++++++++------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index 333ec61a8d..a5c8525019 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -14,6 +14,7 @@ import com.openbankproject.commons.ExecutionContext.Implicits.global import com.openbankproject.commons.util.ApiVersion import com.tesobe.CacheKeyFromArguments import net.liftweb.common.Box +import net.liftweb.db.DB import net.liftweb.mapper.{Index, _} import net.liftweb.util.Helpers.tryo import org.apache.commons.lang3.StringUtils @@ -91,7 +92,9 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ } private def trueOrFalse(condition: Boolean) = if (condition) sqls"1=1" else sqls"0=1" + private def trueOrFalseString(condition: Boolean) = if (condition) s"1=1" else s"0=1" private def falseOrTrue(condition: Boolean) = if (condition) sqls"0=1" else sqls"1=1" + private def falseOrTrueString(condition: Boolean) = if (condition) s"0=1" else s"1=1" // override def getAllGroupedByUserId(): Map[String, List[APIMetric]] = { // //TODO: do this all at the db level using an actual group by query @@ -217,6 +220,26 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ sqls"${params.head})"+ sqlSingleLine + sqls" and url ${isLikeQuery} LIKE (${params.last}" } } + private def extendLikeQueryString(params: List[String], isLike: Boolean): String = { + val isLikeQuery = if (isLike) s"" else s"NOT" + + if (params.length == 1) + s"'${params.head}'" + else + { + val sqlList: immutable.Seq[String] = for (i <- 1 to (params.length - 2)) yield + { + s" and url ${isLikeQuery} LIKE ('${params(i)}')" + } + + val sqlSingleLine = if (sqlList.length>1) + sqlList.reduce(_+_) + else + s"" + + s"${params.head})"+ sqlSingleLine + s" and url ${isLikeQuery} LIKE ('${params.last}''" + } + } /** @@ -373,7 +396,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val userId = queryParams.collect { case OBPUserId(value) => value }.headOption val url = queryParams.collect { case OBPUrl(value) => value }.headOption val appName = queryParams.collect { case OBPAppName(value) => value }.headOption - val excludeAppNames = queryParams.collect { case OBPExcludeAppNames(value) => value }.headOption + val excludeAppNames: Option[List[String]] = queryParams.collect { case OBPExcludeAppNames(value) => value }.headOption val implementedByPartialFunction = queryParams.collect { case OBPImplementedByPartialFunction(value) => value }.headOption val implementedInVersion = queryParams.collect { case OBPImplementedInVersion(value) => value }.headOption val verb = queryParams.collect { case OBPVerb(value) => value }.headOption @@ -385,47 +408,51 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val limit = queryParams.collect { case OBPLimit(value) => value }.headOption.getOrElse(10) val excludeUrlPatternsList= excludeUrlPatterns.getOrElse(List("")) - val excludeAppNamesNumberList = excludeAppNames.getOrElse(List("")) - val excludeImplementedByPartialFunctionsNumberList = excludeImplementedByPartialFunctions.getOrElse(List("")) + val excludeAppNamesNumberList = excludeAppNames.getOrElse(List("")).map(i => s"'$i'").mkString(",") + val excludeImplementedByPartialFunctionsNumberList = + excludeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") - val excludeUrlPatternsQueries = extendLikeQuery(excludeUrlPatternsList, false) + val excludeUrlPatternsQueries: String = extendLikeQueryString(excludeUrlPatternsList, false) val (dbUrl, _, _) = DBUtil.getDbConnectionParameters val result: List[TopApi] = scalikeDB readOnly { implicit session => // MS SQL server has the specific syntax for limiting number of rows - val msSqlLimit = if (dbUrl.contains("sqlserver")) sqls"TOP ($limit)" else sqls"" + val msSqlLimit = if (dbUrl.contains("sqlserver")) s"TOP ($limit)" else s"" // TODO Make it work in case of Oracle database - val otherDbLimit = if (dbUrl.contains("sqlserver")) sqls"" else sqls"LIMIT $limit" - val sqlResult = - sql"""SELECT ${msSqlLimit} count(*), metric.implementedbypartialfunction, metric.implementedinversion + val otherDbLimit = if (dbUrl.contains("sqlserver")) s"" else s"LIMIT $limit" + val sqlQuery: String = + s"""SELECT ${msSqlLimit} count(*), metric.implementedbypartialfunction, metric.implementedinversion FROM metric WHERE - date_c >= ${new Timestamp(fromDate.get.getTime)} AND - date_c <= ${new Timestamp(toDate.get.getTime)} - AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("")}) - AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) - AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) - AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) - AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) - AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) - AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') - AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesNumberList)) - AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) + date_c >= '${new Timestamp(fromDate.get.getTime)}' AND + date_c <= '${new Timestamp(toDate.get.getTime)}' + AND (${trueOrFalseString(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("null")}) + AND (${trueOrFalseString(userId.isEmpty)} or userid = ${userId.getOrElse("null")}) + AND (${trueOrFalseString(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("null")}) + AND (${trueOrFalseString(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("null")}) + AND (${trueOrFalseString(url.isEmpty)} or url = ${url.getOrElse("null")}) + AND (${trueOrFalseString(appName.isEmpty)} or appname = ${appName.getOrElse("null")}) + AND (${trueOrFalseString(verb.isEmpty)} or verb = ${verb.getOrElse("null")}) + AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(true)))} or userid = null) + AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(false)))} or userid != null) + AND (${trueOrFalseString(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) + AND (${trueOrFalseString(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesNumberList)) + AND (${trueOrFalseString(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) GROUP BY metric.implementedbypartialfunction, metric.implementedinversion ORDER BY count(*) DESC ${otherDbLimit} """.stripMargin - .map( - rs => // Map result to case class - TopApi( - rs.string(1).toInt, - rs.string(2), - rs.string(3)) - ).list.apply() + + val (_, rows) = DB.runQuery(sqlQuery, List()) + val sqlResult = + rows.map { rs => // Map result to case class + TopApi( + rs(0).toInt, + rs(1), + rs(2) + ) + } sqlResult } tryo(result) From 07350c8fbd7a83093a2c3046e413400173d81b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 4 Aug 2023 12:15:08 +0200 Subject: [PATCH 06/18] feature/Add property database_query_timeout_in_seconds --- .../resources/props/sample.props.template | 3 +++ .../main/scala/bootstrap/liftweb/Boot.scala | 21 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 37e90c3774..6f496101cf 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -100,6 +100,9 @@ read_authentication_type_validation_requires_role=false ## enable logging all the database queries in log file #logging.database.queries.enable=true +## enable logging all the database queries in log file +#database_query_timeout_in_seconds= + ##Added Props property_name_prefix, default is OBP_. This adds the prefix only for the system environment property name, eg: db.driver --> OBP_db.driver #system_environment_property_name_prefix=OBP_ diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index d92a482ec5..b10785f090 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -149,7 +149,7 @@ import net.liftweb.util.Helpers._ import net.liftweb.util.{DefaultConnectionIdentifier, Helpers, Props, Schedule, _} import org.apache.commons.io.FileUtils -import scala.concurrent.ExecutionContext +import scala.concurrent.{ExecutionContext, Future} /** * A class that's instantiated early and run. It allows the application @@ -279,6 +279,25 @@ class Boot extends MdcLoggable { } } } + + // Database query timeout + APIUtil.getPropsValue("database_query_timeout_in_seconds").map { timeoutInSeconds => + tryo(timeoutInSeconds.toInt).isDefined match { + case true => + DB.queryTimeout = Full(timeoutInSeconds.toInt) + logger.info(s"Query timeout database_query_timeout_in_seconds is set to ${timeoutInSeconds} seconds") + case false => + logger.error( + s""" + |------------------------------------------------------------------------------------ + |Query timeout database_query_timeout_in_seconds [${timeoutInSeconds}] is not an integer value. + |Actual DB.queryTimeout value: ${DB.queryTimeout} + |------------------------------------------------------------------------------------""".stripMargin) + } + + } + + implicit val formats = CustomJsonFormats.formats LiftRules.statelessDispatch.prepend { case _ if tryo(DB.use(DefaultConnectionIdentifier){ conn => conn}.isClosed).isEmpty => From ada5493f354ae04732c1c63fb3f4519e94584c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 7 Aug 2023 10:32:52 +0200 Subject: [PATCH 07/18] refactor/Remove scalikejdbc in case of get top consumers --- .../scala/code/metrics/MappedMetrics.scala | 87 +++++++++++-------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index a5c8525019..62986fc379 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -95,6 +95,14 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ private def trueOrFalseString(condition: Boolean) = if (condition) s"1=1" else s"0=1" private def falseOrTrue(condition: Boolean) = if (condition) sqls"0=1" else sqls"1=1" private def falseOrTrueString(condition: Boolean) = if (condition) s"0=1" else s"1=1" + + private def sqlFriendly(value : Option[String]): String = { + value.isDefined match { + case true => s"'$value'" + case false => "null" + + } + } // override def getAllGroupedByUserId(): Map[String, List[APIMetric]] = { // //TODO: do this all at the db level using an actual group by query @@ -416,7 +424,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val (dbUrl, _, _) = DBUtil.getDbConnectionParameters - val result: List[TopApi] = scalikeDB readOnly { implicit session => + val result: List[TopApi] = { // MS SQL server has the specific syntax for limiting number of rows val msSqlLimit = if (dbUrl.contains("sqlserver")) s"TOP ($limit)" else s"" // TODO Make it work in case of Oracle database @@ -436,20 +444,20 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ AND (${trueOrFalseString(verb.isEmpty)} or verb = ${verb.getOrElse("null")}) AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(true)))} or userid = null) AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(false)))} or userid != null) - AND (${trueOrFalseString(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalseString(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesNumberList)) - AND (${trueOrFalseString(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) + AND (${trueOrFalseString(excludeUrlPatterns.isEmpty)} or (url NOT LIKE ($excludeUrlPatternsQueries))) + AND (${trueOrFalseString(excludeAppNames.isEmpty)} or appname not in ($excludeAppNamesNumberList)) + AND (${trueOrFalseString(excludeImplementedByPartialFunctions.isEmpty)} or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) GROUP BY metric.implementedbypartialfunction, metric.implementedinversion ORDER BY count(*) DESC ${otherDbLimit} """.stripMargin val (_, rows) = DB.runQuery(sqlQuery, List()) - val sqlResult = + val sqlResult = rows.map { rs => // Map result to case class TopApi( - rs(0).toInt, - rs(1), + rs(0).toInt, + rs(1), rs(2) ) } @@ -485,53 +493,56 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val duration = queryParams.collect { case OBPDuration(value) => value }.headOption val excludeUrlPatterns = queryParams.collect { case OBPExcludeUrlPatterns(value) => value }.headOption val excludeImplementedByPartialFunctions = queryParams.collect { case OBPExcludeImplementedByPartialFunctions(value) => value }.headOption - val limit = queryParams.collect { case OBPLimit(value) => value }.headOption + val limit = queryParams.collect { case OBPLimit(value) => value }.headOption.getOrElse("500") val excludeUrlPatternsList = excludeUrlPatterns.getOrElse(List("")) - val excludeAppNamesList = excludeAppNames.getOrElse(List("")) - val excludeImplementedByPartialFunctionsList = excludeImplementedByPartialFunctions.getOrElse(List("")) + val excludeAppNamesList = excludeAppNames.getOrElse(List("")).map(i => s"'$i'").mkString(",") + val excludeImplementedByPartialFunctionsList = + excludeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") - val excludeUrlPatternsQueries = extendLikeQuery(excludeUrlPatternsList, false) + val excludeUrlPatternsQueries: String = extendLikeQueryString(excludeUrlPatternsList, false) val (dbUrl, _, _) = DBUtil.getDbConnectionParameters // MS SQL server has the specific syntax for limiting number of rows - val msSqlLimit = if (dbUrl.contains("sqlserver")) sqls"TOP ($limit)" else sqls"" + val msSqlLimit = if (dbUrl.contains("sqlserver")) s"TOP ($limit)" else s"" // TODO Make it work in case of Oracle database - val otherDbLimit = if (dbUrl.contains("sqlserver")) sqls"" else sqls"LIMIT $limit" + val otherDbLimit: String = if (dbUrl.contains("sqlserver")) s"" else s"LIMIT $limit" - val result: List[TopConsumer] = scalikeDB readOnly { implicit session => - val sqlResult = - sql"""SELECT ${msSqlLimit} count(*) as count, consumer.id as consumerprimaryid, metric.appname as appname, + val result: List[TopConsumer] = { + val sqlQuery = + s"""SELECT ${msSqlLimit} count(*) as count, consumer.id as consumerprimaryid, metric.appname as appname, consumer.developeremail as email, consumer.consumerid as consumerid FROM metric, consumer WHERE metric.appname = consumer.name - AND date_c >= ${new Timestamp(fromDate.get.getTime)} - AND date_c <= ${new Timestamp(toDate.get.getTime)} - AND (${trueOrFalse(consumerId.isEmpty)} or consumer.consumerid = ${consumerId.getOrElse("")}) - AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) - AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) - AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) - AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) - AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) - AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') - AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) - AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) + AND date_c >= '${new Timestamp(fromDate.get.getTime)}' + AND date_c <= '${new Timestamp(toDate.get.getTime)}' + AND (${trueOrFalseString(consumerId.isEmpty)} or consumer.consumerid = ${sqlFriendly(consumerId)}) + AND (${trueOrFalseString(userId.isEmpty)} or userid = ${sqlFriendly(userId)}) + AND (${trueOrFalseString(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${sqlFriendly(implementedByPartialFunction)}) + AND (${trueOrFalseString(implementedInVersion.isEmpty)} or implementedinversion = ${sqlFriendly(implementedInVersion)}) + AND (${trueOrFalseString(url.isEmpty)} or url = ${sqlFriendly(url)}) + AND (${trueOrFalseString(appName.isEmpty)} or appname = ${sqlFriendly(appName)}) + AND (${trueOrFalseString(verb.isEmpty)} or verb = ${sqlFriendly(verb)}) + AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(true)))} or userid = null) + AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(false)))} or userid != null) + AND (${trueOrFalseString(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) + AND (${trueOrFalseString(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) + AND (${trueOrFalseString(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) GROUP BY appname, consumer.developeremail, consumer.id, consumer.consumerid ORDER BY count DESC ${otherDbLimit} """.stripMargin - .map( - rs => - TopConsumer( - rs.string(1).toInt, - rs.string(5), - rs.string(3), - rs.string(4)) - ).list.apply() + val (_, rows) = DB.runQuery(sqlQuery, List()) + val sqlResult = + rows.map { rs => // Map result to case class + TopConsumer( + rs(0).toInt, + rs(4), + rs(2), + rs(3) + ) + } sqlResult } tryo(result) From 046892bd1e1e1b4f22b06ca16b82311e1015c787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 7 Aug 2023 12:59:56 +0200 Subject: [PATCH 08/18] refactor/Remove scalikejdbc in case of get aggregate metric --- .../scala/code/metrics/MappedMetrics.scala | 181 +++++++----------- 1 file changed, 70 insertions(+), 111 deletions(-) diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index 62986fc379..7395907da0 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -4,23 +4,18 @@ import java.sql.{PreparedStatement, Timestamp} import java.util.Date import java.util.UUID.randomUUID -import code.api.Constant import code.api.cache.Caching import code.api.util._ import code.model.MappedConsumersProvider import code.util.Helper.MdcLoggable import code.util.{MappedUUID, UUIDString} import com.openbankproject.commons.ExecutionContext.Implicits.global -import com.openbankproject.commons.util.ApiVersion import com.tesobe.CacheKeyFromArguments import net.liftweb.common.Box import net.liftweb.db.DB import net.liftweb.mapper.{Index, _} import net.liftweb.util.Helpers.tryo import org.apache.commons.lang3.StringUtils -import scalikejdbc.{ConnectionPool, ConnectionPoolSettings, MultipleConnectionPoolContext} -import scalikejdbc.DB.CPContext -import scalikejdbc.{DB => scalikeDB, _} import scala.collection.immutable import scala.collection.immutable.List @@ -91,10 +86,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ metric.save } - private def trueOrFalse(condition: Boolean) = if (condition) sqls"1=1" else sqls"0=1" - private def trueOrFalseString(condition: Boolean) = if (condition) s"1=1" else s"0=1" - private def falseOrTrue(condition: Boolean) = if (condition) sqls"0=1" else sqls"1=1" - private def falseOrTrueString(condition: Boolean) = if (condition) s"0=1" else s"1=1" + private def trueOrFalse(condition: Boolean): String = if (condition) s"1=1" else s"0=1" + private def falseOrTrue(condition: Boolean): String = if (condition) s"0=1" else s"1=1" private def sqlFriendly(value : Option[String]): String = { value.isDefined match { @@ -207,28 +200,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ } } } - - private def extendLikeQuery(params: List[String], isLike: Boolean) = { - val isLikeQuery = if (isLike) sqls"" else sqls"NOT" - - if (params.length == 1) - sqls"${params.head}" - else - { - val sqlList: immutable.Seq[SQLSyntax] = for (i <- 1 to (params.length - 2)) yield - { - sqls" and url ${isLikeQuery} LIKE ('${params(i)}')" - } - - val sqlSingleLine = if (sqlList.length>1) - sqlList.reduce(_+_) - else - sqls"" - - sqls"${params.head})"+ sqlSingleLine + sqls" and url ${isLikeQuery} LIKE (${params.last}" - } - } - private def extendLikeQueryString(params: List[String], isLike: Boolean): String = { + + private def extendLikeQuery(params: List[String], isLike: Boolean): String = { val isLikeQuery = if (isLike) s"" else s"NOT" if (params.length == 1) @@ -262,24 +235,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ stmt.setString(startLine+i, excludeFiledValues.toList(i)) } } - - /** - * this connection pool context corresponding db.url in default.props - */ - implicit lazy val context: CPContext = { - val settings = ConnectionPoolSettings( - initialSize = 5, - maxSize = 20, - connectionTimeoutMillis = 3000L, - validationQuery = "select 1", - connectionPoolFactoryName = "commons-dbcp2" - ) - val (dbUrl, user, password) = DBUtil.getDbConnectionParameters - val dbName = "DB_NAME" // corresponding props db.url DB - ConnectionPool.add(dbName, dbUrl, user, password, settings) - val connectionPool = ConnectionPool.get(dbName) - MultipleConnectionPoolContext(ConnectionPool.DEFAULT_NAME -> connectionPool) - } + // TODO Cache this as long as fromDate and toDate are in the past (before now) def getAllAggregateMetricsBox(queryParams: List[OBPQueryParam], isNewVersion: Boolean): Box[List[AggregateMetrics]] = { @@ -311,68 +267,71 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val includeImplementedByPartialFunctions = queryParams.collect { case OBPIncludeImplementedByPartialFunctions(value) => value }.headOption val excludeUrlPatternsList= excludeUrlPatterns.getOrElse(List("")) - val excludeAppNamesList = excludeAppNames.getOrElse(List("")) - val excludeImplementedByPartialFunctionsList = excludeImplementedByPartialFunctions.getOrElse(List("")) + val excludeAppNamesList = excludeAppNames.getOrElse(List("")).map(i => s"'$i'").mkString(",") + val excludeImplementedByPartialFunctionsList = + excludeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") val excludeUrlPatternsQueries = extendLikeQuery(excludeUrlPatternsList, false) val includeUrlPatternsList= includeUrlPatterns.getOrElse(List("")) - val includeAppNamesList = includeAppNames.getOrElse(List("")) - val includeImplementedByPartialFunctionsList = includeImplementedByPartialFunctions.getOrElse(List("")) + val includeAppNamesList = includeAppNames.getOrElse(List("")).map(i => s"'$i'").mkString(",") + val includeImplementedByPartialFunctionsList = + includeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") val includeUrlPatternsQueries = extendLikeQuery(includeUrlPatternsList, true) - val includeUrlPatternsQueriesSql = sqls"$includeUrlPatternsQueries" + val includeUrlPatternsQueriesSql = s"$includeUrlPatternsQueries" - val result = scalikeDB readOnly { implicit session => + val result = { val sqlQuery = if(isNewVersion) // in the version, we use includeXxx instead of excludeXxx, the performance should be better. - sql"""SELECT count(*), avg(duration), min(duration), max(duration) + s"""SELECT count(*), avg(duration), min(duration), max(duration) FROM metric - WHERE date_c >= ${new Timestamp(fromDate.get.getTime)} - AND date_c <= ${new Timestamp(toDate.get.getTime)} - AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("")}) - AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) - AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) - AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) - AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) - AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) - AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) + WHERE date_c >= '${new Timestamp(fromDate.get.getTime)}' + AND date_c <= '${new Timestamp(toDate.get.getTime)}' + AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${sqlFriendly(consumerId)}) + AND (${trueOrFalse(userId.isEmpty)} or userid = ${sqlFriendly(userId)}) + AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${sqlFriendly(implementedByPartialFunction)}) + AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${sqlFriendly(implementedInVersion)}) + AND (${trueOrFalse(url.isEmpty)} or url = ${sqlFriendly(url)}) + AND (${trueOrFalse(appName.isEmpty)} or appname = ${sqlFriendly(appName)}) + AND (${trueOrFalse(verb.isEmpty)} or verb = ${sqlFriendly(verb)}) AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') - AND (${trueOrFalse(correlationId.isEmpty)} or correlationId = ${correlationId.getOrElse("")}) + AND (${trueOrFalse(correlationId.isEmpty)} or correlationId = ${sqlFriendly(correlationId)}) AND (${trueOrFalse(includeUrlPatterns.isEmpty) } or (url LIKE ($includeUrlPatternsQueriesSql))) AND (${trueOrFalse(includeAppNames.isEmpty) } or (appname in ($includeAppNamesList))) AND (${trueOrFalse(includeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction in ($includeImplementedByPartialFunctionsList)) """.stripMargin else - sql"""SELECT count(*), avg(duration), min(duration), max(duration) + s"""SELECT count(*), avg(duration), min(duration), max(duration) FROM metric - WHERE date_c >= ${new Timestamp(fromDate.get.getTime)} - AND date_c <= ${new Timestamp(toDate.get.getTime)} - AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("")}) - AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) - AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) - AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) - AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) - AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) - AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) + WHERE date_c >= '${new Timestamp(fromDate.get.getTime)}' + AND date_c <= '${new Timestamp(toDate.get.getTime)}' + AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${sqlFriendly(consumerId)}) + AND (${trueOrFalse(userId.isEmpty)} or userid = ${sqlFriendly(userId)}) + AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${sqlFriendly(implementedByPartialFunction)}) + AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${sqlFriendly(implementedInVersion)}) + AND (${trueOrFalse(url.isEmpty)} or url = ${sqlFriendly(url)}) + AND (${trueOrFalse(appName.isEmpty)} or appname = ${sqlFriendly(appName)}) + AND (${trueOrFalse(verb.isEmpty)} or verb = ${sqlFriendly(verb)}) AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') - AND (${trueOrFalse(correlationId.isEmpty)} or correlationId = ${correlationId.getOrElse("")}) + AND (${trueOrFalse(correlationId.isEmpty)} or correlationId = ${sqlFriendly(correlationId)}) AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) """.stripMargin - logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlQuery --: " +sqlQuery.statement) - val sqlResult = sqlQuery.map( + val (_, rows) = DB.runQuery(sqlQuery, List()) + logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlQuery --: " + sqlQuery) + val sqlResult = rows.map( rs => // Map result to case class AggregateMetrics( - rs.stringOpt(1).map(_.toInt).getOrElse(0), - rs.stringOpt(2).map(avg => "%.2f".format(avg.toDouble).toDouble).getOrElse(0), - rs.stringOpt(3).map(_.toDouble).getOrElse(0), - rs.stringOpt(4).map(_.toDouble).getOrElse(0) + tryo(rs(0).toInt).getOrElse(0), + tryo("%.2f".format(rs(1).toDouble).toDouble).getOrElse(0), + tryo(rs(2).toDouble).getOrElse(0), + tryo(rs(3).toDouble).getOrElse(0) ) - ).list().apply() - logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlResult --: "+sqlResult.toString) + ) + logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlResult --: " + sqlResult) sqlResult } tryo(result) @@ -420,7 +379,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val excludeImplementedByPartialFunctionsNumberList = excludeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") - val excludeUrlPatternsQueries: String = extendLikeQueryString(excludeUrlPatternsList, false) + val excludeUrlPatternsQueries: String = extendLikeQuery(excludeUrlPatternsList, false) val (dbUrl, _, _) = DBUtil.getDbConnectionParameters @@ -435,18 +394,18 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ WHERE date_c >= '${new Timestamp(fromDate.get.getTime)}' AND date_c <= '${new Timestamp(toDate.get.getTime)}' - AND (${trueOrFalseString(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("null")}) - AND (${trueOrFalseString(userId.isEmpty)} or userid = ${userId.getOrElse("null")}) - AND (${trueOrFalseString(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("null")}) - AND (${trueOrFalseString(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("null")}) - AND (${trueOrFalseString(url.isEmpty)} or url = ${url.getOrElse("null")}) - AND (${trueOrFalseString(appName.isEmpty)} or appname = ${appName.getOrElse("null")}) - AND (${trueOrFalseString(verb.isEmpty)} or verb = ${verb.getOrElse("null")}) - AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(true)))} or userid = null) - AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(false)))} or userid != null) - AND (${trueOrFalseString(excludeUrlPatterns.isEmpty)} or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalseString(excludeAppNames.isEmpty)} or appname not in ($excludeAppNamesNumberList)) - AND (${trueOrFalseString(excludeImplementedByPartialFunctions.isEmpty)} or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) + AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("null")}) + AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("null")}) + AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("null")}) + AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("null")}) + AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("null")}) + AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("null")}) + AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("null")}) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = null) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != null) + AND (${trueOrFalse(excludeUrlPatterns.isEmpty)} or (url NOT LIKE ($excludeUrlPatternsQueries))) + AND (${trueOrFalse(excludeAppNames.isEmpty)} or appname not in ($excludeAppNamesNumberList)) + AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty)} or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) GROUP BY metric.implementedbypartialfunction, metric.implementedinversion ORDER BY count(*) DESC ${otherDbLimit} @@ -500,7 +459,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val excludeImplementedByPartialFunctionsList = excludeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") - val excludeUrlPatternsQueries: String = extendLikeQueryString(excludeUrlPatternsList, false) + val excludeUrlPatternsQueries: String = extendLikeQuery(excludeUrlPatternsList, false) val (dbUrl, _, _) = DBUtil.getDbConnectionParameters @@ -517,18 +476,18 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ WHERE metric.appname = consumer.name AND date_c >= '${new Timestamp(fromDate.get.getTime)}' AND date_c <= '${new Timestamp(toDate.get.getTime)}' - AND (${trueOrFalseString(consumerId.isEmpty)} or consumer.consumerid = ${sqlFriendly(consumerId)}) - AND (${trueOrFalseString(userId.isEmpty)} or userid = ${sqlFriendly(userId)}) - AND (${trueOrFalseString(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${sqlFriendly(implementedByPartialFunction)}) - AND (${trueOrFalseString(implementedInVersion.isEmpty)} or implementedinversion = ${sqlFriendly(implementedInVersion)}) - AND (${trueOrFalseString(url.isEmpty)} or url = ${sqlFriendly(url)}) - AND (${trueOrFalseString(appName.isEmpty)} or appname = ${sqlFriendly(appName)}) - AND (${trueOrFalseString(verb.isEmpty)} or verb = ${sqlFriendly(verb)}) - AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(true)))} or userid = null) - AND (${falseOrTrueString(anon.isDefined && anon.equals(Some(false)))} or userid != null) - AND (${trueOrFalseString(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalseString(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) - AND (${trueOrFalseString(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) + AND (${trueOrFalse(consumerId.isEmpty)} or consumer.consumerid = ${sqlFriendly(consumerId)}) + AND (${trueOrFalse(userId.isEmpty)} or userid = ${sqlFriendly(userId)}) + AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${sqlFriendly(implementedByPartialFunction)}) + AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${sqlFriendly(implementedInVersion)}) + AND (${trueOrFalse(url.isEmpty)} or url = ${sqlFriendly(url)}) + AND (${trueOrFalse(appName.isEmpty)} or appname = ${sqlFriendly(appName)}) + AND (${trueOrFalse(verb.isEmpty)} or verb = ${sqlFriendly(verb)}) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = null) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != null) + AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) + AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) + AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) GROUP BY appname, consumer.developeremail, consumer.id, consumer.consumerid ORDER BY count DESC ${otherDbLimit} From c30e9e0292813cae586072ec5e2e23bdb2eb2ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 8 Aug 2023 15:01:28 +0200 Subject: [PATCH 09/18] feature/Bump Oracle driver to ojbbc8-production 23.2.0.0 --- obp-api/pom.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/obp-api/pom.xml b/obp-api/pom.xml index 0cbb7cb5b2..846196e016 100644 --- a/obp-api/pom.xml +++ b/obp-api/pom.xml @@ -116,10 +116,12 @@ postgresql 42.4.3 + com.oracle.database.jdbc - ojdbc8 - 21.5.0.0 + ojdbc8-production + 23.2.0.0 + pom From c44c3e6db1c96203c19a4799735345be3ebd7948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 11 Aug 2023 16:10:58 +0200 Subject: [PATCH 10/18] feature/Add Future with timeout --- .../main/scala/code/api/util/FutureUtil.scala | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 obp-api/src/main/scala/code/api/util/FutureUtil.scala diff --git a/obp-api/src/main/scala/code/api/util/FutureUtil.scala b/obp-api/src/main/scala/code/api/util/FutureUtil.scala new file mode 100644 index 0000000000..4d5a57b413 --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/FutureUtil.scala @@ -0,0 +1,59 @@ +package code.api.util + +import java.util.concurrent.TimeoutException +import java.util.{Timer, TimerTask} + +import scala.concurrent.duration.FiniteDuration +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.language.postfixOps + +object FutureUtil { + + // All Future's that use futureWithTimeout will use the same Timer object + // it is thread safe and scales to thousands of active timers + // The true parameter ensures that timeout timers are daemon threads and do not stop + // the program from shutting down + + val timer: Timer = new Timer(true) + + /** + * Returns the result of the provided future within the given time or a timeout exception, whichever is first + * This uses Java Timer which runs a single thread to handle all futureWithTimeouts and does not block like a + * Thread.sleep would + * @param future Caller passes a future to execute + * @param timeout Time before we return a Timeout exception instead of future's outcome + * @return Future[T] + */ + def futureWithTimeout[T](future : Future[T], timeout : FiniteDuration)(implicit ec: ExecutionContext): Future[T] = { + + // Promise will be fulfilled with either the callers Future or the timer task if it times out + var p = Promise[T] + + // and a Timer task to handle timing out + + val timerTask = new TimerTask() { + def run() : Unit = { + p.tryFailure(new TimeoutException("Request Timeout")) + } + } + + // Set the timeout to check in the future + timer.schedule(timerTask, timeout.toMillis) + + future.map { + a => + if(p.trySuccess(a)) { + timerTask.cancel() + } + } + .recover { + case e: Exception => + if(p.tryFailure(e)) { + timerTask.cancel() + } + } + + p.future + } + +} From 55d38686f7b40a3e39f14913f30fdc1afa9b9acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 14 Aug 2023 13:41:46 +0200 Subject: [PATCH 11/18] feature/Add ErrorMessages.requestTimeout --- obp-api/src/main/scala/code/api/util/APIUtil.scala | 5 +++++ obp-api/src/main/scala/code/api/util/ErrorMessages.scala | 4 ++++ obp-api/src/main/scala/code/api/util/FutureUtil.scala | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index ab44b9f34b..189cd7fdcc 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -807,6 +807,9 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ def check401(message: String): Boolean = { message.contains(extractErrorMessageCode(UserNotLoggedIn)) } + def check408(message: String): Boolean = { + message.contains(extractErrorMessageCode(requestTimeout)) + } val (code, responseHeaders) = message match { case msg if check401(msg) => @@ -816,6 +819,8 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ (401, getHeaders() ::: headers.list ::: addHeader) case msg if check403(msg) => (403, getHeaders() ::: headers.list) + case msg if check408(msg) => + (408, getHeaders() ::: headers.list) case _ => (httpCode, getHeaders() ::: headers.list) } diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 7954296e17..7bc3a62f79 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -41,6 +41,10 @@ object ErrorMessages { val NoValidElasticsearchIndicesConfigured = "OBP-00011: No elasticsearch indices are allowed on this instance. Please set es.warehouse.allowed.indices = index1,index2 (or = ALL for all). " val CustomerFirehoseNotAllowedOnThisInstance = "OBP-00012: Customer firehose is not allowed on this instance. Please set allow_customer_firehose = true in props files. " + // Exceptions (OBP-01XXX) ------------------------------------------------> + val requestTimeout = "OBP-01000: Request Timeout. " + // <------------------------------------------------ Exceptions (OBP-01XXX) + // WebUiProps Exceptions (OBP-08XXX) val InvalidWebUiProps = "OBP-08001: Incorrect format of name." val WebUiPropsNotFound = "OBP-08002: WebUi props not found. Please specify a valid value for WEB_UI_PROPS_ID." diff --git a/obp-api/src/main/scala/code/api/util/FutureUtil.scala b/obp-api/src/main/scala/code/api/util/FutureUtil.scala index 4d5a57b413..baf4b648fa 100644 --- a/obp-api/src/main/scala/code/api/util/FutureUtil.scala +++ b/obp-api/src/main/scala/code/api/util/FutureUtil.scala @@ -33,7 +33,7 @@ object FutureUtil { val timerTask = new TimerTask() { def run() : Unit = { - p.tryFailure(new TimeoutException("Request Timeout")) + p.tryFailure(new TimeoutException(ErrorMessages.requestTimeout)) } } From 7d165f9214cb66308936fc923456f734e71c4b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 14 Aug 2023 16:42:18 +0200 Subject: [PATCH 12/18] refactor/Make Thread.sleep non blocking at Waiting For Godot endpoint --- .../src/main/scala/code/api/v5_1_0/APIMethods510.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala index 2fa5abe4a9..f7342cbf37 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala @@ -114,11 +114,11 @@ trait APIMethods510 { cc => for { httpParams <- NewStyle.function.extractHttpParamsFromUrl(cc.url) - } yield { - val sleep: String = httpParams.filter(_.name == "sleep").headOption + sleep: String = httpParams.filter(_.name == "sleep").headOption .map(_.values.headOption.getOrElse("0")).getOrElse("0") - val sleepInMillis: Long = tryo(sleep.trim.toLong).getOrElse(0) - Thread.sleep(sleepInMillis) + sleepInMillis: Long = tryo(sleep.trim.toLong).getOrElse(0) + _ <- Future(Thread.sleep(sleepInMillis)) + } yield { (JSONFactory510.waitingForGodot(sleepInMillis), HttpCode.`200`(cc.callContext)) } } From 4d8dfa66f0341d6e5ee5bd43d2f67eb06577dc1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 15 Aug 2023 10:22:43 +0200 Subject: [PATCH 13/18] feature/Add props regardng endpoint timeouts --- obp-api/src/main/resources/props/sample.props.template | 5 +++++ obp-api/src/main/scala/code/api/util/FutureUtil.scala | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 6f496101cf..e89c384e80 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -103,6 +103,11 @@ read_authentication_type_validation_requires_role=false ## enable logging all the database queries in log file #database_query_timeout_in_seconds= +## Define endpoint timeouts in seconds +short_endpoint_timeout = 1 +medium_endpoint_timeout = 7 +long_endpoint_timeout = 60 + ##Added Props property_name_prefix, default is OBP_. This adds the prefix only for the system environment property name, eg: db.driver --> OBP_db.driver #system_environment_property_name_prefix=OBP_ diff --git a/obp-api/src/main/scala/code/api/util/FutureUtil.scala b/obp-api/src/main/scala/code/api/util/FutureUtil.scala index baf4b648fa..79c20bba61 100644 --- a/obp-api/src/main/scala/code/api/util/FutureUtil.scala +++ b/obp-api/src/main/scala/code/api/util/FutureUtil.scala @@ -6,6 +6,7 @@ import java.util.{Timer, TimerTask} import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.postfixOps +import scala.concurrent.duration._ object FutureUtil { @@ -15,6 +16,8 @@ object FutureUtil { // the program from shutting down val timer: Timer = new Timer(true) + + val defaultTimeout: Int = APIUtil.getPropsAsIntValue(nameOfProperty = "medium_endpoint_timeout", 7) /** * Returns the result of the provided future within the given time or a timeout exception, whichever is first @@ -24,7 +27,7 @@ object FutureUtil { * @param timeout Time before we return a Timeout exception instead of future's outcome * @return Future[T] */ - def futureWithTimeout[T](future : Future[T], timeout : FiniteDuration)(implicit ec: ExecutionContext): Future[T] = { + def futureWithTimeout[T](future : Future[T], timeout : FiniteDuration = defaultTimeout seconds)(implicit ec: ExecutionContext): Future[T] = { // Promise will be fulfilled with either the callers Future or the timer task if it times out var p = Promise[T] From 1a0bcc46b305be0368c1cd2ea75a4ab7e1173c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 15 Aug 2023 10:25:22 +0200 Subject: [PATCH 14/18] feature/Add endpoint timeout in case of new style --- obp-api/src/main/scala/code/api/util/APIUtil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 189cd7fdcc..a96d3ad3d7 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -2956,7 +2956,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * @return */ implicit def scalaFutureToBoxedJsonResponse[T](scf: OBPReturnType[T])(implicit m: Manifest[T]): Box[JsonResponse] = { - futureToBoxedResponse(scalaFutureToLaFuture(scf)) + futureToBoxedResponse(scalaFutureToLaFuture(FutureUtil.futureWithTimeout(scf))) } From 86af44b4ee5406e26f7a69f249c4b0ba53b8b749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 15 Aug 2023 11:48:33 +0200 Subject: [PATCH 15/18] feature/Tweak request timeout error message --- obp-api/src/main/scala/code/api/util/ErrorMessages.scala | 2 +- obp-api/src/main/scala/code/api/util/FutureUtil.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 7bc3a62f79..12c536f260 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -42,7 +42,7 @@ object ErrorMessages { val CustomerFirehoseNotAllowedOnThisInstance = "OBP-00012: Customer firehose is not allowed on this instance. Please set allow_customer_firehose = true in props files. " // Exceptions (OBP-01XXX) ------------------------------------------------> - val requestTimeout = "OBP-01000: Request Timeout. " + val requestTimeout = "OBP-01000: Request Timeout. The OBP API decided to return a timeout. This is probably because a backend service did not respond in time. " // <------------------------------------------------ Exceptions (OBP-01XXX) // WebUiProps Exceptions (OBP-08XXX) diff --git a/obp-api/src/main/scala/code/api/util/FutureUtil.scala b/obp-api/src/main/scala/code/api/util/FutureUtil.scala index 79c20bba61..0483499927 100644 --- a/obp-api/src/main/scala/code/api/util/FutureUtil.scala +++ b/obp-api/src/main/scala/code/api/util/FutureUtil.scala @@ -17,7 +17,7 @@ object FutureUtil { val timer: Timer = new Timer(true) - val defaultTimeout: Int = APIUtil.getPropsAsIntValue(nameOfProperty = "medium_endpoint_timeout", 7) + val defaultTimeout: Int = APIUtil.getPropsAsIntValue(nameOfProperty = "long_endpoint_timeout", 60) /** * Returns the result of the provided future within the given time or a timeout exception, whichever is first From c5f7494c8ef25cc66059565256bb963f3e6d0682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 16 Aug 2023 05:22:33 +0200 Subject: [PATCH 16/18] feature/Add Connectin: close header regarding 408 Request Timeout --- obp-api/src/main/scala/code/api/constant/constant.scala | 1 + obp-api/src/main/scala/code/api/util/APIUtil.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 8b70623d67..892bd5e9eb 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -102,6 +102,7 @@ object ResponseHeader { final lazy val `WWW-Authenticate` = "WWW-Authenticate" final lazy val ETag = "ETag" final lazy val `Cache-Control` = "Cache-Control" + final lazy val Connection = "Connection" } object BerlinGroup extends Enumeration { diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index a96d3ad3d7..17a0f07b5e 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -820,7 +820,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ case msg if check403(msg) => (403, getHeaders() ::: headers.list) case msg if check408(msg) => - (408, getHeaders() ::: headers.list) + (408, getHeaders() ::: headers.list ::: List((ResponseHeader.Connection, "close"))) case _ => (httpCode, getHeaders() ::: headers.list) } From 20b7581568b8bed4e4c5138280b710e39a43e9b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 16 Aug 2023 05:53:19 +0200 Subject: [PATCH 17/18] feature/Tweak props regardng endpoint timeouts --- obp-api/src/main/resources/props/sample.props.template | 8 ++++---- obp-api/src/main/scala/code/api/util/FutureUtil.scala | 10 ++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index e89c384e80..b6efb92cce 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -103,10 +103,10 @@ read_authentication_type_validation_requires_role=false ## enable logging all the database queries in log file #database_query_timeout_in_seconds= -## Define endpoint timeouts in seconds -short_endpoint_timeout = 1 -medium_endpoint_timeout = 7 -long_endpoint_timeout = 60 +## Define endpoint timeouts in miliseconds +short_endpoint_timeout = 1000 +medium_endpoint_timeout = 7000 +long_endpoint_timeout = 60000 ##Added Props property_name_prefix, default is OBP_. This adds the prefix only for the system environment property name, eg: db.driver --> OBP_db.driver #system_environment_property_name_prefix=OBP_ diff --git a/obp-api/src/main/scala/code/api/util/FutureUtil.scala b/obp-api/src/main/scala/code/api/util/FutureUtil.scala index 0483499927..14a91ac0f1 100644 --- a/obp-api/src/main/scala/code/api/util/FutureUtil.scala +++ b/obp-api/src/main/scala/code/api/util/FutureUtil.scala @@ -3,10 +3,8 @@ package code.api.util import java.util.concurrent.TimeoutException import java.util.{Timer, TimerTask} -import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.postfixOps -import scala.concurrent.duration._ object FutureUtil { @@ -17,17 +15,17 @@ object FutureUtil { val timer: Timer = new Timer(true) - val defaultTimeout: Int = APIUtil.getPropsAsIntValue(nameOfProperty = "long_endpoint_timeout", 60) + val defaultTimeout: Long = APIUtil.getPropsAsLongValue(nameOfProperty = "long_endpoint_timeout", 60L * 1000L) /** * Returns the result of the provided future within the given time or a timeout exception, whichever is first * This uses Java Timer which runs a single thread to handle all futureWithTimeouts and does not block like a * Thread.sleep would * @param future Caller passes a future to execute - * @param timeout Time before we return a Timeout exception instead of future's outcome + * @param timeoutInMillis Time before we return a Timeout exception instead of future's outcome * @return Future[T] */ - def futureWithTimeout[T](future : Future[T], timeout : FiniteDuration = defaultTimeout seconds)(implicit ec: ExecutionContext): Future[T] = { + def futureWithTimeout[T](future : Future[T], timeoutInMillis : Long = defaultTimeout)(implicit ec: ExecutionContext): Future[T] = { // Promise will be fulfilled with either the callers Future or the timer task if it times out var p = Promise[T] @@ -41,7 +39,7 @@ object FutureUtil { } // Set the timeout to check in the future - timer.schedule(timerTask, timeout.toMillis) + timer.schedule(timerTask, timeoutInMillis) future.map { a => From ac53cb9c98163e5dc40bbeccff81049a4a47b09b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 16 Aug 2023 11:15:03 +0200 Subject: [PATCH 18/18] feature/Override general request timeout at an endpoint --- .../src/main/scala/code/api/constant/constant.scala | 4 ++++ obp-api/src/main/scala/code/api/util/APIUtil.scala | 3 ++- .../src/main/scala/code/api/util/FutureUtil.scala | 12 ++++++++---- .../main/scala/code/api/v5_1_0/APIMethods510.scala | 3 +++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 892bd5e9eb..b0707099f5 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -14,6 +14,10 @@ object Constant extends MdcLoggable { final val limit = 500 } + final val shortEndpointTimeoutInMillis = APIUtil.getPropsAsLongValue(nameOfProperty = "short_endpoint_timeout", 1L * 1000L) + final val mediumEndpointTimeoutInMillis = APIUtil.getPropsAsLongValue(nameOfProperty = "medium_endpoint_timeout", 7L * 1000L) + final val longEndpointTimeoutInMillis = APIUtil.getPropsAsLongValue(nameOfProperty = "long_endpoint_timeout", 60L * 1000L) + final val h2DatabaseDefaultUrlValue = "jdbc:h2:mem:OBPTest_H2_v2.1.214;NON_KEYWORDS=VALUE;DB_CLOSE_DELAY=10" final val HostName = APIUtil.getPropsValue("hostname").openOrThrowException(ErrorMessages.HostnameNotSpecified) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 17a0f07b5e..24ce46b399 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -116,6 +116,7 @@ import org.apache.commons.lang3.StringUtils import java.security.AccessControlException import java.util.regex.Pattern +import code.api.util.FutureUtil.EndpointTimeout import code.etag.MappedETag import code.users.Users import net.liftweb.mapper.By @@ -2955,7 +2956,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * @tparam T * @return */ - implicit def scalaFutureToBoxedJsonResponse[T](scf: OBPReturnType[T])(implicit m: Manifest[T]): Box[JsonResponse] = { + implicit def scalaFutureToBoxedJsonResponse[T](scf: OBPReturnType[T])(implicit t: EndpointTimeout, m: Manifest[T]): Box[JsonResponse] = { futureToBoxedResponse(scalaFutureToLaFuture(FutureUtil.futureWithTimeout(scf))) } diff --git a/obp-api/src/main/scala/code/api/util/FutureUtil.scala b/obp-api/src/main/scala/code/api/util/FutureUtil.scala index 14a91ac0f1..7202c2f028 100644 --- a/obp-api/src/main/scala/code/api/util/FutureUtil.scala +++ b/obp-api/src/main/scala/code/api/util/FutureUtil.scala @@ -3,6 +3,8 @@ package code.api.util import java.util.concurrent.TimeoutException import java.util.{Timer, TimerTask} +import code.api.Constant + import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.postfixOps @@ -15,17 +17,19 @@ object FutureUtil { val timer: Timer = new Timer(true) - val defaultTimeout: Long = APIUtil.getPropsAsLongValue(nameOfProperty = "long_endpoint_timeout", 60L * 1000L) + case class EndpointTimeout(inMillis: Long) + + implicit val defaultTimeout: EndpointTimeout = EndpointTimeout(Constant.longEndpointTimeoutInMillis) /** * Returns the result of the provided future within the given time or a timeout exception, whichever is first * This uses Java Timer which runs a single thread to handle all futureWithTimeouts and does not block like a * Thread.sleep would * @param future Caller passes a future to execute - * @param timeoutInMillis Time before we return a Timeout exception instead of future's outcome + * @param timeout Time before we return a Timeout exception instead of future's outcome * @return Future[T] */ - def futureWithTimeout[T](future : Future[T], timeoutInMillis : Long = defaultTimeout)(implicit ec: ExecutionContext): Future[T] = { + def futureWithTimeout[T](future : Future[T])(implicit timeout : EndpointTimeout, ec: ExecutionContext): Future[T] = { // Promise will be fulfilled with either the callers Future or the timer task if it times out var p = Promise[T] @@ -39,7 +43,7 @@ object FutureUtil { } // Set the timeout to check in the future - timer.schedule(timerTask, timeoutInMillis) + timer.schedule(timerTask, timeout.inMillis) future.map { a => diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala index f7342cbf37..21366d3fd5 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala @@ -1,11 +1,13 @@ package code.api.v5_1_0 +import code.api.Constant import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.{apiCollectionJson400, apiCollectionsJson400, apiInfoJson400, postApiCollectionJson400, revokedConsentJsonV310, _} import code.api.util.APIUtil._ import code.api.util.ApiRole._ import code.api.util.ApiTag._ import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFound, InvalidJsonFormat, UnknownError, UserNotFoundByUserId, UserNotLoggedIn, _} +import code.api.util.FutureUtil.EndpointTimeout import code.api.util.NewStyle.HttpCode import code.api.util._ import code.api.v2_0_0.{EntitlementJSONs, JSONFactory200} @@ -112,6 +114,7 @@ trait APIMethods510 { lazy val waitingForGodot: OBPEndpoint = { case "waiting-for-godot" :: Nil JsonGet _ => { cc => + implicit val timeout = EndpointTimeout(Constant.mediumEndpointTimeoutInMillis) // Set endpoint timeout explicitly for { httpParams <- NewStyle.function.extractHttpParamsFromUrl(cc.url) sleep: String = httpParams.filter(_.name == "sleep").headOption