Merge pull request #2317 from hongwei1/develop

feature/addedOboCacheForResourceDocs

obp-api/src/main/resources/props/sample.props.template

@@ -91,12 +91,6 @@ authTypeValidation.cache.ttl.seconds=36
 ## no 0 value will cause new ConnectorMethod will works after that seconds
 connectorMethod.cache.ttl.seconds=40
 
-## swagger file should not generated for every request, this is a time-to-live in seconds for the generated swagger of OBP api,
-## this value also represent how many seconds before the new endpoints will be shown after upload a new DynamicEntity.
-## So if you want the new endpoints shown timely, set this value to a small number.
-dynamicResourceDocsObp.cache.ttl.seconds=3600
-staticResourceDocsObp.cache.ttl.seconds=3600
-createLocalisedResourceDocJson.cache.ttl.seconds=3600
 
 ## This can change the behavior of  `Get Resource Docs`/`Get API Glossary`. If we set it to `true`, OBP will check the authentication and CanReadResourceDoc/CanReadGlossary Role
 # the default value is false, so the `Get Resource Docs`/`Get API Glossary` is anonymous as default. 

obp-api/src/main/scala/code/api/cache/Caching.scala

@@ -1,6 +1,7 @@
 package code.api.cache
 
-import code.api.util.RateLimitingUtil
+import code.api.Constant._
+import code.api.cache.Redis.jedis
 import code.util.Helper.MdcLoggable
 import com.softwaremill.macmemo.{Cache, MemoCacheBuilder, MemoizeParams}
 
@@ -84,4 +85,74 @@ object Caching extends MdcLoggable {
     }
   }
 
+  def getLocalisedResourceDocCache(key: String) = {
+    if(Redis.isRedisAvailable())
+      jedis.get(LOCALISED_RESOURCE_DOC_PREFIX + key)
+    else 
+      null
+  }
+
+  def setLocalisedResourceDocCache(key:String, value: String)={
+    if (Redis.isRedisAvailable())
+      jedis.set(LOCALISED_RESOURCE_DOC_PREFIX+key,value)
+    else
+      null
+  }
+
+  def getDynamicResourceDocCache(key: String) = {
+    if (Redis.isRedisAvailable())
+      jedis.get(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key)
+    else
+      null
+  }
+
+  def setDynamicResourceDocCache(key:String, value: String)={
+    if (Redis.isRedisAvailable())
+      jedis.set(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value)
+    else
+      null
+  }
+
+  def getStaticResourceDocCache(key: String) = {
+    if (Redis.isRedisAvailable())
+      jedis.get(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key)
+    else
+      null
+  }
+
+  def setStaticResourceDocCache(key:String, value: String)={
+    if (Redis.isRedisAvailable())
+      jedis.set(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value)
+    else
+      null
+  }
+
+  def getAllResourceDocCache(key: String) = {
+    if (Redis.isRedisAvailable())
+      jedis.get(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX + key)
+    else
+      null
+  }
+
+  def setAllResourceDocCache(key:String, value: String)={
+    if (Redis.isRedisAvailable())
+      jedis.set(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value)
+    else
+      null
+  }
+
+  def getStaticSwaggerDocCache(key: String) = {
+    if (Redis.isRedisAvailable())
+      jedis.get(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX + key)
+    else
+      null
+  }
+
+  def setStaticSwaggerDocCache(key:String, value: String)={
+    if (Redis.isRedisAvailable())
+      jedis.set(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX+key,value)
+    else
+      null
+  }
+
 }

obp-api/src/main/scala/code/api/cache/Redis.scala

@@ -22,14 +22,15 @@ object Redis extends MdcLoggable {
 
   def isRedisAvailable() = {
     try {
-      val uuid = APIUtil.generateUUID()
-      jedis.connect()
-      jedis.set(uuid, "10")
-      jedis.exists(uuid) == true
+      val status = jedis.isConnected
+      if (!status) {
+        logger.warn("------------| Redis is not connected|------------")
+      }
+      status
     } catch {
       case e: Throwable =>
-        logger.warn("------------| Redis.isRedisAvailable |------------")
-        logger.warn(e)
+        logger.error("------------| Redis throw exception|------------")
+        logger.error(e)
         false
     }
   }
@@ -54,7 +55,7 @@ object Redis extends MdcLoggable {
       tryDecode match {
         case Success(v) => v.asInstanceOf[T]
         case Failure(e) =>
-          println(e)
+          logger.error(e)
           "NONE".asInstanceOf[T]
       }
     }

obp-api/src/main/scala/code/api/constant/constant.scala

@@ -87,6 +87,13 @@ object Constant extends MdcLoggable {
 
   final val PARAM_LOCALE = "locale"
   final val PARAM_TIMESTAMP = "_timestamp_"
+
+
+  final val LOCALISED_RESOURCE_DOC_PREFIX = "rd_localised_"
+  final val DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_dynamic_"
+  final val STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_static_"
+  final val ALL_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_all_"
+  final val STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX = "swagger_static_"
 }
 
 

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -4872,6 +4872,4 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
       )
   }
 
-    val createLocalisedResourceDocJsonTTL : Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt
-
 }

obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala

@@ -1,6 +1,7 @@
 package code.api.util
 
 import code.api.APIFailureNewStyle
+import code.api.cache.Redis.{isRedisAvailable, jedis}
 import code.api.util.APIUtil.fullBoxOrException
 import code.api.util.ErrorMessages.TooManyRequests
 import code.api.util.RateLimitingJson.CallLimit
@@ -70,28 +71,9 @@ object RateLimitingJson {
 
 object RateLimitingUtil extends MdcLoggable {
   import code.api.util.RateLimitingPeriod._
-
-  val port = APIUtil.getPropsAsIntValue("redis_port", 6379)
-  val url = APIUtil.getPropsValue("redis_address", "127.0.0.1")
 
   def useConsumerLimits = APIUtil.getPropsAsBoolValue("use_consumer_limits", false)
 
-  lazy val jedis = new Jedis(url, port)
-
-  def isRedisAvailable() = {
-    try {
-      val uuid = APIUtil.generateUUID()
-      jedis.connect()
-      jedis.set(uuid, "10")
-      jedis.exists(uuid) == true
-    } catch {
-      case e : Throwable =>
-        logger.warn("------------| RateLimitUtil.isRedisAvailable |------------")
-        logger.warn(e)
-        false
-    }
-  }
-
   private def createUniqueKey(consumerKey: String, period: LimitCallPeriod) = consumerKey + RateLimitingPeriod.toString(period)
 
   private def underConsumerLimits(consumerKey: String, period: LimitCallPeriod, limit: Long): Boolean = {

obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala

@@ -3,7 +3,7 @@ package code.api.v1_4_0
 import code.api.berlin.group.v1_3.JvalueCaseClass
 import code.api.cache.Caching
 import java.util.Date
-import code.api.util.APIUtil.{EmptyBody, PrimaryDataBody, ResourceDoc, createLocalisedResourceDocJsonTTL}
+import code.api.util.APIUtil.{EmptyBody, PrimaryDataBody, ResourceDoc}
 import code.api.util.ApiTag.ResourceDocTag
 import code.api.util.Glossary.glossaryItems
 import code.api.util.{APIUtil, ApiRole, ConnectorField, CustomJsonFormats, ExampleValue, I18NUtil, PegdownOptions}
@@ -17,13 +17,15 @@ import com.openbankproject.commons.util.{EnumValue, JsonUtils, OBPEnumeration, R
 import net.liftweb.common.Full
 import net.liftweb.json
 import net.liftweb.json.Extraction.decompose
-import net.liftweb.json.{Formats, JDouble, JInt, JString}
+import net.liftweb.json.{Extraction, Formats, JDouble, JInt, JString}
 import net.liftweb.json.JsonAST.{JArray, JBool, JNothing, JObject, JValue}
 import net.liftweb.util.StringHelpers
 import code.util.Helper.MdcLoggable
+import com.github.dwickern.macros.NameOf.nameOf
 import com.tesobe.{CacheKeyFromArguments, CacheKeyOmit}
 import org.apache.commons.lang3.StringUtils
 import scalacache.memoization.cacheKeyExclude
+
 import java.util.regex.Pattern
 import java.lang.reflect.Field
 import java.util.UUID.randomUUID
@@ -408,7 +410,7 @@ object JSONFactory1_4_0 extends MdcLoggable{
       case Some(ConnectorField(value, _)) => value
       case _ =>
         //The ExampleValue are not totally finished, lots of fields are missing here. so we first hide them. and show them in the log
-        logger.debug(s"getExampleFieldValue: there is no $exampleValueFieldName variable in ExampleValue object")
+        logger.trace(s"getExampleFieldValue: there is no $exampleValueFieldName variable in ExampleValue object")
         parameter
     }
   }
@@ -517,35 +519,19 @@ object JSONFactory1_4_0 extends MdcLoggable{
 
     jsonFieldsDescription.mkString(jsonTitleType,"","\n")
   }
+
   //cache key will only contain "operationId + locale"
+
   def createLocalisedResourceDocJsonCached(
     operationId: String, // this will be in the cacheKey
     locale: Option[String],// this will be in the cacheKey
-    @CacheKeyOmit resourceDocUpdatedTags: ResourceDoc,
-    @CacheKeyOmit isVersion4OrHigher:Boolean,
-    @CacheKeyOmit urlParametersI18n:String ,
-    @CacheKeyOmit jsonRequestBodyFieldsI18n:String,
-    @CacheKeyOmit jsonResponseBodyFieldsI18n:String
+    resourceDocUpdatedTags: ResourceDoc,
+    isVersion4OrHigher:Boolean,
+    urlParametersI18n:String ,
+    jsonRequestBodyFieldsI18n:String,
+    jsonResponseBodyFieldsI18n:String
   ): ResourceDocJson = {
-    /**
-     * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)"
-     * is just a temporary value field with UUID values in order to prevent any ambiguity.
-     * The real value will be assigned by Macro during compile time at this line of a code:
-     * https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/tesobe/CacheKeyFromArgumentsMacro.scala#L49
-     */
-    var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
-    CacheKeyFromArguments.buildCacheKey {
-      Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(createLocalisedResourceDocJsonTTL second) {
-        // There are multiple flavours of markdown. For instance, original markdown emphasises underscores (surrounds _ with (<em>))
-        // But we don't want to have to escape underscores (\_) in our documentation
-        // Thus we use a flavour of markdown that ignores underscores in words. (Github markdown does this too)
-        // We return html rather than markdown to the consumer so they don't have to bother with these questions.
-
-        //Here area some endpoints, which should not be added the description:
-        // 1st: Dynamic entity endpoint,
-        // 2rd: Dynamic endpoint endpoints,
-        // 3rd: all the user created endpoints,
-        val fieldsDescription =
+      val fieldsDescription =
         if (resourceDocUpdatedTags.tags.toString.contains("Dynamic-Entity")
           || resourceDocUpdatedTags.tags.toString.contains("Dynamic-Endpoint")
           || resourceDocUpdatedTags.roles.toString.contains("DynamicEntity")
@@ -603,26 +589,34 @@ object JSONFactory1_4_0 extends MdcLoggable{
 
         logger.trace(s"createLocalisedResourceDocJsonCached value is $resourceDoc")
         resourceDoc
-
-      }
     }
-  }
+
 
   def createLocalisedResourceDocJson(rd: ResourceDoc, isVersion4OrHigher:Boolean, locale: Option[String], urlParametersI18n:String ,jsonRequestBodyFieldsI18n:String, jsonResponseBodyFieldsI18n:String) : ResourceDocJson = {
     // We MUST recompute all resource doc values due to translation via Web UI props --> now need to wait $createLocalisedResourceDocJsonTTL seconds
     val userDefinedEndpointTags = getAllEndpointTagsBox(rd.operationId).map(endpointTag =>ResourceDocTag(endpointTag.tagName))
     val resourceDocWithUserDefinedEndpointTags: ResourceDoc = rd.copy(tags = userDefinedEndpointTags++ rd.tags)
 
-    logger.trace(s"createLocalisedResourceDocJsonCached key is ${resourceDocWithUserDefinedEndpointTags.operationId + resourceDocWithUserDefinedEndpointTags.operationId}")
-    createLocalisedResourceDocJsonCached(
-      resourceDocWithUserDefinedEndpointTags.operationId,
-      locale: Option[String],
-      resourceDocWithUserDefinedEndpointTags,
-      isVersion4OrHigher: Boolean,
-      urlParametersI18n: String,
-      jsonRequestBodyFieldsI18n: String,
-      jsonResponseBodyFieldsI18n: String
-    )
+    val cacheKey = (resourceDocWithUserDefinedEndpointTags.operationId + locale).intern()
+    val cacheValueFromRedis = Caching.getLocalisedResourceDocCache(cacheKey)
+
+    if(cacheValueFromRedis != null){
+      json.parse(cacheValueFromRedis).extract[ResourceDocJson] 
+    }else{
+      val resourceDocJson = createLocalisedResourceDocJsonCached(
+        resourceDocWithUserDefinedEndpointTags.operationId,
+        locale: Option[String],
+        resourceDocWithUserDefinedEndpointTags,
+        isVersion4OrHigher: Boolean,
+        urlParametersI18n: String,
+        jsonRequestBodyFieldsI18n: String,
+        jsonResponseBodyFieldsI18n: String
+      )
+      val jsonString = json.compactRender(Extraction.decompose(resourceDocJson))
+      Caching.setLocalisedResourceDocCache(cacheKey,jsonString)
+
+      resourceDocJson
+    }
 
   }
 

obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala

@@ -2,12 +2,13 @@ package code.api.v3_1_0
 
 import code.api.Constant
 import code.api.Constant.{SYSTEM_OWNER_VIEW_ID, localIdentityProvider}
+
 import java.text.SimpleDateFormat
 import java.util.UUID
 import java.util.regex.Pattern
-
 import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._
 import code.api.ResourceDocs1_4_0.{MessageDocsSwaggerDefinitions, ResourceDocsAPIMethodsUtil, SwaggerDefinitionsJSON, SwaggerJSONFactory}
+import code.api.cache.Redis
 import code.api.util.APIUtil.{getWebUIPropsPairs, _}
 import code.api.util.ApiRole._
 import code.api.util.ApiTag._
@@ -1242,7 +1243,7 @@ trait APIMethods310 {
           for {
             (_, callContext) <- anonymousAccess(cc)
             rateLimiting <- NewStyle.function.tryons("", 400, callContext) {
-              val isRedisAvailable = RateLimitingUtil.isRedisAvailable()
+              val isRedisAvailable = Redis.isRedisAvailable()
               val isActive = if (RateLimitingUtil.useConsumerLimits && isRedisAvailable) true else false
               RateLimiting(RateLimitingUtil.useConsumerLimits, "REDIS", isRedisAvailable, isActive)
             }

obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala

@@ -11828,8 +11828,6 @@ trait APIMethods400 extends MdcLoggable {
       "/management/endpoints/OPERATION_ID/tags",
       "Create System Level Endpoint Tag",
       s"""Create System Level Endpoint Tag
-         |
-         |Note: Resource Docs are cached, TTL is ${createLocalisedResourceDocJsonTTL}
          |
          |""".stripMargin,
       endpointTagJson400,
@@ -11872,8 +11870,6 @@ trait APIMethods400 extends MdcLoggable {
       "/management/endpoints/OPERATION_ID/tags/ENDPOINT_TAG_ID",
       "Update System Level Endpoint Tag",
       s"""Update System Level Endpoint Tag, you can only update the tag_name here, operation_id can not be updated.
-         |
-         |Note: Resource Docs are cached, TTL is ${createLocalisedResourceDocJsonTTL}
          |
          |""".stripMargin,
       endpointTagJson400,
@@ -11980,8 +11976,6 @@ trait APIMethods400 extends MdcLoggable {
       "/management/banks/BANK_ID/endpoints/OPERATION_ID/tags",
       "Create Bank Level Endpoint Tag",
       s"""Create Bank Level Endpoint Tag
-         |
-         |Note: Resource Docs are cached, TTL is ${createLocalisedResourceDocJsonTTL}
          |
          |
          |""".stripMargin,
@@ -12027,8 +12021,6 @@ trait APIMethods400 extends MdcLoggable {
       "/management/banks/BANK_ID/endpoints/OPERATION_ID/tags/ENDPOINT_TAG_ID",
       "Update Bank Level Endpoint Tag",
       s"""Update Endpoint Tag, you can only update the tag_name here, operation_id can not be updated.
-         |
-         |Note: Resource Docs are cached, TTL is ${createLocalisedResourceDocJsonTTL}
          |
          |""".stripMargin,
       endpointTagJson400,