Merge pull request #2319 from hongwei1/develop

feature/enhanced the obpSimpleCache

obp-api/src/main/resources/props/sample.props.template

@@ -91,6 +91,12 @@ authTypeValidation.cache.ttl.seconds=36
 ## no 0 value will cause new ConnectorMethod will works after that seconds
 connectorMethod.cache.ttl.seconds=40
 
+## swagger file should not generated for every request, this is a time-to-live in seconds for the generated swagger of OBP api,
+## this value also represent how many seconds before the new endpoints will be shown after upload a new DynamicEntity.
+## So if you want the new endpoints shown timely, set this value to a small number.
+dynamicResourceDocsObp.cache.ttl.seconds=3600
+staticResourceDocsObp.cache.ttl.seconds=3600
+createLocalisedResourceDocJson.cache.ttl.seconds=3600
 
 ## This can change the behavior of  `Get Resource Docs`/`Get API Glossary`. If we set it to `true`, OBP will check the authentication and CanReadResourceDoc/CanReadGlossary Role
 # the default value is false, so the `Get Resource Docs`/`Get API Glossary` is anonymous as default. 

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -44,6 +44,7 @@ import code.api.ResourceDocs1_4_0._
 import code.api._
 import code.api.attributedefinition.AttributeDefinition
 import code.api.builder.APIBuilder_Connector
+import code.api.cache.Redis
 import code.api.util.APIUtil.{enableVersionIfAllowed, errorJsonResponse, getPropsValue, gitCommit}
 import code.api.util._
 import code.api.util.migration.Migration
@@ -364,7 +365,7 @@ class Boot extends MdcLoggable {
 //    }
 
     LiftRules.unloadHooks.append(APIUtil.vendor.closeAllConnections_! _)
-
+    LiftRules.unloadHooks.append(Redis.jedisPoolDestroy _)
 //    LiftRules.statelessDispatch.prepend {
 //      case _ if tryo(DB.use(DefaultConnectionIdentifier){ conn => conn}.isClosed).isEmpty =>
 //        Props.mode match {

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala

@@ -1,6 +1,6 @@
 package code.api.ResourceDocs1_4_0
 
-import code.api.Constant.PARAM_LOCALE
+import code.api.Constant.{GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, PARAM_LOCALE}
 import java.util.UUID.randomUUID
 
 import code.api.OBPRestHelper
@@ -402,6 +402,10 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
          |
          |See the Resource Doc endpoint for more information.
          |
+         |Note: Dynamic Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds
+         |      Static Resource Docs are cached, TTL is ${GET_STATIC_RESOURCE_DOCS_TTL} seconds
+         |
+         |
          |Following are more examples:
          |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp
          |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?tags=Account,Bank
@@ -549,8 +553,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
                 case Some(DYNAMIC) =>{
                   val cacheValueFromRedis = Caching.getDynamicResourceDocCache(cacheKey)
                   val dynamicDocs: Box[JValue] =
-                    if (cacheValueFromRedis != null) {
-                      Full(json.parse(cacheValueFromRedis))
+                    if (cacheValueFromRedis.isDefined) {
+                      Full(json.parse(cacheValueFromRedis.get))
                     } else {
                       val resourceDocJsonJValue = getResourceDocsObpDynamicCached(tags, partialFunctions, locale, None, isVersion4OrHigher)
                       val jsonString = json.compactRender(resourceDocJsonJValue)
@@ -565,8 +569,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
                   val cacheValueFromRedis = Caching.getStaticResourceDocCache(cacheKey)
 
                   val dynamicDocs: Box[JValue] =
-                    if (cacheValueFromRedis != null) {
-                      Full(json.parse(cacheValueFromRedis))
+                    if (cacheValueFromRedis.isDefined) {
+                      Full(json.parse(cacheValueFromRedis.get))
                     } else {
                       val resourceDocJsonJValue = getStaticResourceDocsObpCached(requestedApiVersionString, tags, partialFunctions, locale, isVersion4OrHigher)
                       val jsonString = json.compactRender(resourceDocJsonJValue)
@@ -580,8 +584,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
                   val cacheValueFromRedis = Caching.getAllResourceDocCache(cacheKey)
 
                   val dynamicDocs: Box[JValue] =
-                    if (cacheValueFromRedis != null) {
-                      Full(json.parse(cacheValueFromRedis))
+                    if (cacheValueFromRedis.isDefined) {
+                      Full(json.parse(cacheValueFromRedis.get))
                     } else {
                       val resourceDocJsonJValue = getAllResourceDocsObpCached(requestedApiVersionString, tags, partialFunctions, locale, contentParam, isVersion4OrHigher)
                       val jsonString = json.compactRender(resourceDocJsonJValue)
@@ -637,8 +641,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
 
             json <- NewStyle.function.tryons(s"$UnknownError Can not create dynamic resource docs.", 400, callContext) {
               val cacheValueFromRedis = Caching.getDynamicResourceDocCache(cacheKey)
-              if (cacheValueFromRedis != null) {
-                json.parse(cacheValueFromRedis)
+              if (cacheValueFromRedis.isDefined) {
+                json.parse(cacheValueFromRedis.get)
               } else {
                 val resourceDocJsonJValue = getResourceDocsObpDynamicCached(tags, partialFunctions, locale, None, false)
                 val jsonString = json.compactRender(resourceDocJsonJValue)
@@ -674,6 +678,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
          |
          |See the Resource Doc endpoint for more information.
          |
+         | Note: Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds
+         | 
          |Following are more examples:
          |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger
          |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger?tags=Account,Bank
@@ -705,8 +711,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
               val cacheValueFromRedis = Caching.getStaticSwaggerDocCache(cacheKey)
 
               val dynamicDocs: JValue =
-                if (cacheValueFromRedis != null) {
-                  json.parse(cacheValueFromRedis)
+                if (cacheValueFromRedis.isDefined) {
+                  json.parse(cacheValueFromRedis.get)
                 } else {
                   val resourceDocJsonJValue = getResourceDocsSwaggerCached(requestedApiVersionString, resourceDocTags, partialFunctions)
                   val jsonString = json.compactRender(resourceDocJsonJValue)

obp-api/src/main/scala/code/api/cache/Caching.scala

@@ -1,7 +1,9 @@
 package code.api.cache
 
 import code.api.Constant._
-import code.api.cache.Redis.jedis
+import code.api.JedisMethod
+import code.api.cache.Redis.use
+import code.api.util.APIUtil
 import code.util.Helper.MdcLoggable
 import com.softwaremill.macmemo.{Cache, MemoCacheBuilder, MemoizeParams}
 
@@ -15,12 +17,6 @@ object Caching extends MdcLoggable {
     (cacheKey, ttl) match {
       case (_, t) if t == Duration.Zero  => // Just forwarding a call
         f
-      case (Some(_), _) if !Redis.isRedisAvailable() => // Redis is NOT available. Warn via log file and forward the call
-        logger.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
-        logger.warn("! Redis is NOT available at this instance !")
-        logger.warn("! Caching is skipped                      !")
-        logger.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
-        f
       case (Some(_), _) => // Caching a call
         Redis.memoizeSyncWithRedis(cacheKey)(ttl)(f)
       case _  => // Just forwarding a call
@@ -33,12 +29,6 @@ object Caching extends MdcLoggable {
     (cacheKey, ttl) match {
       case (_, t) if t == Duration.Zero  => // Just forwarding a call
         f
-      case (Some(_), _) if !Redis.isRedisAvailable() => // Redis is NOT available. Warn via log file and forward the call
-        logger.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
-        logger.warn("! Redis is NOT available at this instance !")
-        logger.warn("! Caching is skipped                      !")
-        logger.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
-        f
       case (Some(_), _) => // Caching a call
         Redis.memoizeWithRedis(cacheKey)(ttl)(f)
       case _  => // Just forwarding a call
@@ -86,73 +76,43 @@ object Caching extends MdcLoggable {
   }
 
   def getLocalisedResourceDocCache(key: String) = {
-    if(Redis.isRedisAvailable())
-      jedis.get(LOCALISED_RESOURCE_DOC_PREFIX + key)
-    else 
-      null
+    use(JedisMethod.GET, (LOCALISED_RESOURCE_DOC_PREFIX + key).intern(), Some(CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL)) 
   }
-
-  def setLocalisedResourceDocCache(key:String, value: String)={
-    if (Redis.isRedisAvailable())
-      jedis.set(LOCALISED_RESOURCE_DOC_PREFIX+key,value)
-    else
-      null
+
+  def setLocalisedResourceDocCache(key:String, value: String)=  {
+    use(JedisMethod.SET, (LOCALISED_RESOURCE_DOC_PREFIX+key).intern(), Some(CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL), Some(value))
   }
 
   def getDynamicResourceDocCache(key: String) = {
-    if (Redis.isRedisAvailable())
-      jedis.get(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key)
-    else
-      null
+    use(JedisMethod.GET, (DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key).intern(), Some(GET_DYNAMIC_RESOURCE_DOCS_TTL))
   }
 
-  def setDynamicResourceDocCache(key:String, value: String)={
-    if (Redis.isRedisAvailable())
-      jedis.set(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value)
-    else
-      null
+  def setDynamicResourceDocCache(key:String, value: String)= {
+    use(JedisMethod.SET, (DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key).intern(), Some(GET_DYNAMIC_RESOURCE_DOCS_TTL), Some(value))
   }
 
   def getStaticResourceDocCache(key: String) = {
-    if (Redis.isRedisAvailable())
-      jedis.get(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key)
-    else
-      null
+    use(JedisMethod.GET, (STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key).intern(), Some(GET_STATIC_RESOURCE_DOCS_TTL))
   }
 
-  def setStaticResourceDocCache(key:String, value: String)={
-    if (Redis.isRedisAvailable())
-      jedis.set(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value)
-    else
-      null
+  def setStaticResourceDocCache(key:String, value: String)= {
+    use(JedisMethod.SET, (STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key).intern(), Some(GET_STATIC_RESOURCE_DOCS_TTL), Some(value))
   }
 
   def getAllResourceDocCache(key: String) = {
-    if (Redis.isRedisAvailable())
-      jedis.get(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX + key)
-    else
-      null
+    use(JedisMethod.GET, (ALL_RESOURCE_DOC_CACHE_KEY_PREFIX + key).intern(), Some(GET_DYNAMIC_RESOURCE_DOCS_TTL))
   }
 
-  def setAllResourceDocCache(key:String, value: String)={
-    if (Redis.isRedisAvailable())
-      jedis.set(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value)
-    else
-      null
+  def setAllResourceDocCache(key:String, value: String)= {
+    use(JedisMethod.SET, (ALL_RESOURCE_DOC_CACHE_KEY_PREFIX+key).intern(), Some(GET_DYNAMIC_RESOURCE_DOCS_TTL), Some(value))
   }
 
   def getStaticSwaggerDocCache(key: String) = {
-    if (Redis.isRedisAvailable())
-      jedis.get(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX + key)
-    else
-      null
+    use(JedisMethod.GET, (STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX + key).intern(), Some(GET_STATIC_RESOURCE_DOCS_TTL))
   }
 
-  def setStaticSwaggerDocCache(key:String, value: String)={
-    if (Redis.isRedisAvailable())
-      jedis.set(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX+key,value)
-    else
-      null
+  def setStaticSwaggerDocCache(key:String, value: String)= {
+    use(JedisMethod.SET, (STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX+key).intern(), Some(GET_STATIC_RESOURCE_DOCS_TTL), Some(value))
   }
 
 }

obp-api/src/main/scala/code/api/cache/Redis.scala

@@ -1,9 +1,10 @@
 package code.api.cache
 
+import code.api.JedisMethod
 import code.api.util.APIUtil
 import code.util.Helper.MdcLoggable
 import com.openbankproject.commons.ExecutionContext.Implicits.global
-import redis.clients.jedis.Jedis
+import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
 import scalacache.memoization.{cacheKeyExclude, memoize, memoizeSync}
 import scalacache.{Flags, ScalaCache}
 import scalacache.redis.RedisCache
@@ -18,21 +19,76 @@ object Redis extends MdcLoggable {
   val url = APIUtil.getPropsValue("cache.redis.url", "127.0.0.1")
   val port = APIUtil.getPropsAsIntValue("cache.redis.port", 6379)
 
-  lazy val jedis = new Jedis(url, port)
+  final val poolConfig = new JedisPoolConfig()
+  poolConfig.setMaxTotal(128)
+  poolConfig.setMaxIdle(128)
+  poolConfig.setMinIdle(16)
+  poolConfig.setTestOnBorrow(true)
+  poolConfig.setTestOnReturn(true)
+  poolConfig.setTestWhileIdle(true)
+  poolConfig.setMinEvictableIdleTimeMillis(30*60*1000)
+  poolConfig.setTimeBetweenEvictionRunsMillis(30*60*1000)
+  poolConfig.setNumTestsPerEvictionRun(3)
+  poolConfig.setBlockWhenExhausted(true)
 
-  def isRedisAvailable() = {
-    try {
-      val status = jedis.isConnected
-      if (!status) {
-        logger.warn("------------| Redis is not connected|------------")
+  def jedisPoolDestroy: Unit = jedisPool.destroy()
+  val jedisPool = new JedisPool(poolConfig,url, port, 4000)
+
+  /**
+   * this is the help method, which can be used to auto close all the jedisConnection
+   * 
+   * @param method can only be "get" or "set" 
+   * @param key the cache key
+   * @param ttlSeconds the ttl is option. 
+   *            if ttl == None, this means value will be cached forver 
+   *            if ttl == Some(0), this means turn off the cache, do not use cache at all
+   *            if ttl == Some(Int), this mean the cache will be only cached for ttl seconds
+   * @param value the cache value.
+   *              
+   * @return
+   */
+  def use(method:JedisMethod.Value, key:String, ttlSeconds: Option[Int] = None, value:Option[String] = None) : Option[String] = {
+
+    //we will get the connection from jedisPool later, and will always close it in the finally clause.
+    var jedisConnection = None:Option[Jedis]
+
+    if(ttlSeconds.equals(Some(0))){ // set ttl = 0, we will totally turn off the cache
+      None
+    }else{
+      try {
+        jedisConnection = Some(jedisPool.getResource())
+
+        val redisResult = if (method ==JedisMethod.EXISTS) {
+          jedisConnection.head.exists(key).toString
+        }else if (method == JedisMethod.FLUSHDB) {
+          jedisConnection.head.flushDB.toString
+        }else if (method == JedisMethod.INCR) {
+          jedisConnection.head.incr(key).toString
+        }else if (method == JedisMethod.TTL) {
+          jedisConnection.head.ttl(key).toString
+        }else if (method == JedisMethod.DELETE) {
+          jedisConnection.head.del(key).toString
+        }else if (method ==JedisMethod.GET) {
+          jedisConnection.head.get(key)
+        } else if(method ==JedisMethod.SET && value.isDefined){
+          if (ttlSeconds.isDefined) {//if set ttl, call `setex` method to set the expired seconds.
+            jedisConnection.head.setex(key, ttlSeconds.get, value.get).toString
+          } else {//if do not set ttl, call `set` method, the cache will be forever. 
+            jedisConnection.head.set(key, value.get).toString
+          }
+        } else {// the use()method parameters need to be set properly, it missing value in set, then will throw the exception. 
+          throw new RuntimeException("Please check the Redis.use parameters, if the method == set, the value can not be None !!!")
+        }
+        //change the null to Option 
+        APIUtil.stringOrNone(redisResult)
+      } catch {
+        case e: Throwable =>
+          throw new RuntimeException(e)
+      } finally {
+        if (jedisConnection.isDefined && jedisConnection.get != null)
+          jedisConnection.map(_.close())
       }
-      status
-    } catch {
-      case e: Throwable =>
-        logger.error("------------| Redis throw exception|------------")
-        logger.error(e)
-        false
-    }
+    } 
   }
 
   implicit val scalaCache = ScalaCache(RedisCache(url, port))

obp-api/src/main/scala/code/api/constant/constant.scala

@@ -94,11 +94,21 @@ object Constant extends MdcLoggable {
   final val STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_static_"
   final val ALL_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_all_"
   final val STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX = "swagger_static_"
+  val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt
+  val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt
+  val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt
+
 }
 
 
 
 
+object JedisMethod extends Enumeration {
+  type JedisMethod = Value
+  val GET, SET, EXISTS, DELETE, TTL, INCR, FLUSHDB= Value
+}
+
+
 object ChargePolicy extends Enumeration {
   type ChargePolicy = Value
   val SHARED, SENDER, RECEIVER = Value