Merge pull request #2264 from hongwei1/develop

refactor/added the basic guard for all url parameters

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -125,7 +125,7 @@ import code.transactionrequests.{MappedTransactionRequest, MappedTransactionRequ
 import code.usercustomerlinks.MappedUserCustomerLink
 import code.userlocks.UserLocks
 import code.users._
-import code.util.Helper.{MdcLoggable, SILENCE_IS_GOLDEN}
+import code.util.Helper.{MdcLoggable, ObpS, SILENCE_IS_GOLDEN}
 import code.util.{Helper, HydraUtil}
 import code.validation.JsonSchemaValidation
 import code.views.Views
@@ -631,7 +631,7 @@ class Boot extends MdcLoggable {
 
         // Check to see if the user explicitly requests a new locale 
         // In case it's true we use that value to set up a new cookie value
-        S.param(PARAM_LOCALE) match {
+        ObpS.param(PARAM_LOCALE) match {
           case Full(requestedLocale) if requestedLocale != null && APIUtil.checkShortString(requestedLocale)==SILENCE_IS_GOLDEN => {
             val computedLocale: Locale = I18NUtil.computeLocale(requestedLocale)
             AuthUser.getCurrentUser.map(_.user.userPrimaryKey.value) match {

obp-api/src/main/scala/code/api/OBPRestHelper.scala

@@ -41,7 +41,7 @@ import code.api.v5_0_0.OBPAPI5_0_0
 import code.api.v5_1_0.OBPAPI5_1_0
 import code.loginattempts.LoginAttempt
 import code.model.dataAccess.AuthUser
-import code.util.Helper.MdcLoggable
+import code.util.Helper.{MdcLoggable, ObpS}
 import com.alibaba.ttl.TransmittableThreadLocal
 import com.openbankproject.commons.model.ErrorMessage
 import com.openbankproject.commons.util.{ApiVersion, ReflectUtils, ScannedApiVersion}
@@ -341,7 +341,7 @@ trait OBPRestHelper extends RestHelper with MdcLoggable {
     val body: Box[String] = getRequestBody(S.request)
     val implementedInVersion = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).view
     val verb = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).requestType.method
-    val url = URLDecoder.decode(S.uriAndQueryString.getOrElse(""),"UTF-8")
+    val url = URLDecoder.decode(ObpS.uriAndQueryString.getOrElse(""),"UTF-8")
     val correlationId = getCorrelationId()
     val reqHeaders = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).request.headers
     val remoteIpAddress = getRemoteIpAddress()

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala

@@ -21,7 +21,7 @@ import code.api.v3_0_0.OBPAPI3_0_0
 import code.api.v3_1_0.OBPAPI3_1_0
 import code.api.v4_0_0.{APIMethods400, OBPAPI4_0_0}
 import code.apicollectionendpoint.MappedApiCollectionEndpointsProvider
-import code.util.Helper.MdcLoggable
+import code.util.Helper.{MdcLoggable, ObpS}
 import com.github.dwickern.macros.NameOf.nameOf
 import com.openbankproject.commons.model.{BankId, ListResult, User}
 import com.openbankproject.commons.model.enums.ContentParam.{ALL, DYNAMIC, STATIC}
@@ -250,7 +250,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
       CacheKeyFromArguments.buildCacheKey {
         Caching.memoizeSyncWithProvider (Some(cacheKey.toString())) (getStaticResourceDocsTTL second) {
-          logger.debug(s"Generating OBP Resource Docs requestedApiVersion is $requestedApiVersionString")
+          logger.debug(s"Generating OBP-getStaticResourceDocsObpCached requestedApiVersion is $requestedApiVersionString")
           val requestedApiVersion  = ApiVersionUtils.valueOf(requestedApiVersionString)
 
           val resourceDocJson = resourceDocsToResourceDocJson(getResourceDocsList(requestedApiVersion), resourceDocTags, partialFunctionNames, isVersion4OrHigher, locale)
@@ -284,7 +284,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
       CacheKeyFromArguments.buildCacheKey {
         Caching.memoizeSyncWithProvider (Some(cacheKey.toString())) (getStaticResourceDocsTTL second) {
-          logger.debug(s"Generating OBP Resource Docs requestedApiVersion is $requestedApiVersionString")
+          logger.debug(s"Generating getAllResourceDocsObpCached-Docs requestedApiVersion is $requestedApiVersionString")
           val requestedApiVersion  = ApiVersionUtils.valueOf(requestedApiVersionString)
 
           val dynamicDocs = allDynamicResourceDocs
@@ -723,7 +723,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
       CacheKeyFromArguments.buildCacheKey {
         Caching.memoizeSyncWithProvider (Some(cacheKey.toString())) (getStaticResourceDocsTTL second) {
-          logger.debug(s"Generating Swagger requestedApiVersion is $requestedApiVersionString")
+          logger.debug(s"Generating Swagger-getResourceDocsSwaggerCached requestedApiVersion is $requestedApiVersionString")
 
           Box.tryo(ApiVersionUtils.valueOf(requestedApiVersionString)) match {
             case Full(requestedApiVersion) =>
@@ -941,7 +941,7 @@ object ResourceDocsAPIMethodsUtil extends MdcLoggable{
 
   def getParams() : (Option[List[ResourceDocTag]], Option[List[String]], Option[String], Option[ContentParam], Option[String], Option[String]) = {
 
-    val rawTagsParam = S.param("tags")
+    val rawTagsParam = ObpS.param("tags")
 
 
     val tags: Option[List[ResourceDocTag]] =
@@ -965,7 +965,7 @@ object ResourceDocsAPIMethodsUtil extends MdcLoggable{
     logger.debug(s"tagsOption is $tags")
 
     // So we can produce a reduced list of resource docs to prevent manual editing of swagger files.
-    val rawPartialFunctionNames = S.param("functions")
+    val rawPartialFunctionNames = ObpS.param("functions")
 
     val partialFunctionNames: Option[List[String]] =
       rawPartialFunctionNames match {
@@ -987,23 +987,23 @@ object ResourceDocsAPIMethodsUtil extends MdcLoggable{
       }
     logger.debug(s"partialFunctionNames is $partialFunctionNames")
 
-    val locale = S.param(PARAM_LOCALE).or(S.param("language")) // we used language before, so keep it there. 
+    val locale = ObpS.param(PARAM_LOCALE).or(ObpS.param("language")) // we used language before, so keep it there. 
     logger.debug(s"locale is $locale")
 
     // So we can produce a reduced list of resource docs to prevent manual editing of swagger files.
     val contentParam = for {
-      x <- S.param("content")
+      x <- ObpS.param("content")
       y <- stringToContentParam(x)
     } yield y
     logger.debug(s"content is $contentParam")
 
     val apiCollectionIdParam = for {
-      x <- S.param("api-collection-id")
+      x <- ObpS.param("api-collection-id")
     } yield x
     logger.debug(s"apiCollectionIdParam is $apiCollectionIdParam")
 
     val cacheModifierParam = for {
-      x <- S.param("cache-modifier")
+      x <- ObpS.param("cache-modifier")
     } yield x
     logger.debug(s"cacheModifierParam is $cacheModifierParam")
 

obp-api/src/main/scala/code/api/oauth1.0.scala

@@ -39,7 +39,7 @@ import code.model.{Consumer, TokenType, UserX}
 import code.nonce.Nonces
 import code.token.Tokens
 import code.users.Users
-import code.util.Helper.MdcLoggable
+import code.util.Helper.{MdcLoggable, ObpS}
 import com.openbankproject.commons.model.User
 import net.liftweb.common._
 import net.liftweb.http.rest.RestHelper
@@ -282,7 +282,7 @@ object OAuthHandshake extends RestHelper with MdcLoggable {
 
     val sRequest = S.request
     val urlParams: Map[String, List[String]] = sRequest.map(_.params).getOrElse(Map.empty)
-    val sUri = S.uri
+    val sUri = ObpS.uri
 
     //are all the necessary OAuth parameters present?
     val missingParams = missingOAuthParameters(parameters,requestType)
@@ -547,7 +547,7 @@ object OAuthHandshake extends RestHelper with MdcLoggable {
 
     val sRequest = S.request
     val urlParams: Map[String, List[String]] = sRequest.map(_.params).getOrElse(Map.empty)
-    val sUri = S.uri
+    val sUri = ObpS.uri
 
     // Please note that after this point S.request for instance cannot be used directly
     // If you need it later assign it to some variable and pass it

obp-api/src/main/scala/code/api/openidconnect.scala

@@ -27,7 +27,6 @@ TESOBE (http://www.tesobe.com/)
 package code.api
 
 import java.net.HttpURLConnection
-
 import code.api.util.APIUtil._
 import code.api.util.{APIUtil, AfterApiAuth, ErrorMessages, JwtUtil}
 import code.consumer.Consumers
@@ -37,7 +36,7 @@ import code.model.dataAccess.AuthUser
 import code.snippet.OpenIDConnectSessionState
 import code.token.{OpenIDConnectToken, TokensOpenIDConnect}
 import code.users.Users
-import code.util.Helper.MdcLoggable
+import code.util.Helper.{MdcLoggable, ObpS}
 import com.openbankproject.commons.model.User
 import com.openbankproject.commons.util.{ApiVersion,ApiVersionStatus}
 import javax.net.ssl.HttpsURLConnection
@@ -182,8 +181,8 @@ object OpenIdConnect extends OBPRestHelper with MdcLoggable {
   }
 
   private def extractParams(s: S): (String, String, String) = {
-    val code = s.param("code")
-    val state = s.param("state")
+    val code = ObpS.param("code")
+    val state = ObpS.param("state")
     val sessionState = OpenIDConnectSessionState.get
     (code.getOrElse(""), state.getOrElse("0"), sessionState.map(_.toString).getOrElse("1"))
   }

obp-api/src/main/scala/code/api/sandbox/SandboxApiCalls.scala

@@ -29,7 +29,7 @@
 //        logger.debug("Hello from v1.0 data-import")
 //        for{
 //          correctToken <- APIUtil.getPropsValue("sandbox_data_import_secret") ~> APIFailure("Data import is disabled for this API instance.", 403)
-//          providedToken <- S.param("secret_token") ~> APIFailure("secret_token parameter required", 403)
+//          providedToken <- ObpS.param("secret_token") ~> APIFailure("secret_token parameter required", 403)
 //          tokensMatch <- Helper.booleanToBox(providedToken == correctToken) ~> APIFailure("incorrect secret token", 403)
 //          importData <- tryo{json.extract[SandboxDataImport]} ?~ ErrorMessages.InvalidJsonFormat
 //          importWorked <- OBPDataImport.importer.vend.importData(importData)

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -71,7 +71,7 @@ import code.model.dataAccess.AuthUser
 import code.sanitycheck.SanityCheck
 import code.scope.Scope
 import code.usercustomerlinks.UserCustomerLink
-import code.util.Helper.{MdcLoggable, SILENCE_IS_GOLDEN}
+import code.util.Helper.{MdcLoggable, ObpS, SILENCE_IS_GOLDEN}
 import code.util.{Helper, JsonSchemaUtil}
 import code.views.{MapperViews, Views}
 import code.webuiprops.MappedWebUiPropsProvider.getWebUiPropsValue
@@ -419,7 +419,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
       val implementedInVersion = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).view
       //(GET, POST etc.) --S.request.get.requestType.method
       val verb = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).requestType.method
-      val url = S.uriAndQueryString.getOrElse("")
+      val url = ObpS.uriAndQueryString.getOrElse("")
       val correlationId = getCorrelationId()
 
       //execute saveMetric in future, as we do not need to know result of operation
@@ -711,10 +711,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
           case Full(h) =>
             Full(h)
           case _ =>
-            S.param(nameOfSpellingParam())
+            ObpS.param(nameOfSpellingParam())
         }
       case _ =>
-        S.param(nameOfSpellingParam())
+        ObpS.param(nameOfSpellingParam())
     }
   }
 
@@ -886,6 +886,31 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
       case _ => false
     }
   }
+
+  def basicUrlValidation(urlString: String): Boolean = {
+    //in scala test - org.scalatest.FeatureSpecLike.scenario: 
+    // redirectUrl = http%3A%2F%2Flocalhost%3A8016%3Foauth_token%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461
+    // URLDecoder.decode(urlString,"UTF-8")-->http://localhost:8016?oauth_token=EBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK&oauth_verifier=63461
+    val regex =
+      """((([A-Za-z]{3,9}:(?:\/\/)?)(?:[-;:&=\+\$,\w]+@)?[A-Za-z0-9.-]+(:[0-9]+)?|(?:www.|[-;:&=\+\$,\w]+@)[A-Za-z0-9.-]+)((?:\/[\+~%\/.\w-_]*)?\??(?:[-\+=&;%@.\w_]*)#?(?:[\w]*))?)""".r
+    val decodeUrlValue = URLDecoder.decode(urlString, "UTF-8").trim()
+    decodeUrlValue match {
+      case regex(_*) if (decodeUrlValue.length <= 2048) => true
+      case _ => false
+    }
+  }
+
+
+  /** only  A-Z, a-z, 0-9,-,_,. =, & and max length <= 2048  */
+  def basicUriAndQueryStringValidation(urlString: String): Boolean = {
+    val regex =
+      """^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?""".r
+    val decodeUrlValue = URLDecoder.decode(urlString, "UTF-8").trim()
+    decodeUrlValue match {
+      case regex(_*) if (decodeUrlValue.length <= 2048) => true
+      case _ => false
+    }
+  }
 
 
 
@@ -2988,7 +3013,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
     val body: Box[String] = getRequestBody(S.request)
     val implementedInVersion = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).view
     val verb = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).requestType.method
-    val url = URLDecoder.decode(S.uriAndQueryString.getOrElse(""),"UTF-8")
+    val url = URLDecoder.decode(ObpS.uriAndQueryString.getOrElse(""),"UTF-8")
     val correlationId = getCorrelationId()
     val reqHeaders = S.request.openOrThrowException(attemptedToOpenAnEmptyBox).request.headers
     val remoteIpAddress = getRemoteIpAddress()
@@ -3534,7 +3559,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
     val brandParameter = "brand"
 
     // Use brand in parameter (query or form)
-    val brand: Option[String] = S.param(brandParameter) match {
+    val brand: Option[String] = ObpS.param(brandParameter) match {
       case Full(value) => {
         // If found, and has a valid format, set the session.
         if (isValidID(value)) {

obp-api/src/main/scala/code/api/util/I18NUtil.scala

@@ -1,7 +1,7 @@
 package code.api.util
 
 import code.api.Constant.PARAM_LOCALE
-import code.util.Helper.SILENCE_IS_GOLDEN
+import code.util.Helper.{MdcLoggable, ObpS, SILENCE_IS_GOLDEN}
 
 import java.util.{Date, Locale}
 
@@ -29,7 +29,7 @@ object I18NUtil extends MdcLoggable {
   def currentLocale() : Locale = {
     // Cookie name
     val localeCookieName = "SELECTED_LOCALE"
-    S.param(PARAM_LOCALE) match {
+    ObpS.param(PARAM_LOCALE) match {
       // 1st choice: Use query parameter as a source of truth if any
       case Full(requestedLocale) if requestedLocale != null && APIUtil.checkShortString(requestedLocale) == SILENCE_IS_GOLDEN => {
         val computedLocale = I18NUtil.computeLocale(requestedLocale)

obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala

@@ -27,7 +27,7 @@ import code.scope.Scope
 import code.search.elasticsearchWarehouse
 import code.users.Users
 import code.util.Helper
-import code.util.Helper.{booleanToBox, booleanToFuture}
+import code.util.Helper.{booleanToBox, booleanToFuture,ObpS}
 import code.views.Views
 import code.views.system.ViewDefinition
 import com.github.dwickern.macros.NameOf.nameOf
@@ -1338,12 +1338,12 @@ trait APIMethods300 {
       case "banks" :: BankId(bankId) :: "branches" :: Nil JsonGet _ => {
         cc => {
           implicit val ec = EndpointContext(Some(cc))
-          val limit = S.param("limit")
-          val offset = S.param("offset")
-          val city = S.param("city")
-          val withinMetersOf = S.param("withinMetersOf")
-          val nearLatitude = S.param("nearLatitude")
-          val nearLongitude = S.param("nearLongitude")
+          val limit = ObpS.param("limit")
+          val offset = ObpS.param("offset")
+          val city = ObpS.param("city")
+          val withinMetersOf = ObpS.param("withinMetersOf")
+          val nearLatitude = ObpS.param("nearLatitude")
+          val nearLongitude = ObpS.param("nearLongitude")
           for {
             (_, callContext) <- getBranchesIsPublic match {
                 case false => authenticatedAccess(cc)
@@ -1471,8 +1471,8 @@ trait APIMethods300 {
       case "banks" :: BankId(bankId) :: "atms" :: Nil JsonGet req => {
         cc => {
           implicit val ec = EndpointContext(Some(cc))
-          val limit = S.param("limit")
-          val offset = S.param("offset")
+          val limit = ObpS.param("limit")
+          val offset = ObpS.param("offset")
           for {
             (_, callContext) <- getAtmsIsPublic match {
                 case false => authenticatedAccess(cc)

obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala

@@ -13,7 +13,7 @@ import code.api.util.ApiRole._
 import code.api.util.ApiTag._
 import code.api.util.ErrorMessages.{BankAccountNotFound, _}
 import code.api.util.ExampleValue._
-import code.api.util.FutureUtil.{EndpointContext}
+import code.api.util.FutureUtil.EndpointContext
 import code.api.util.NewStyle.HttpCode
 import code.api.util._
 import code.api.v1_2_1.{JSONFactory, RateLimiting}
@@ -39,6 +39,7 @@ import code.ratelimiting.RateLimitingDI
 import code.userlocks.{UserLocks, UserLocksProvider}
 import code.users.Users
 import code.util.Helper
+import code.util.Helper.ObpS
 import code.views.Views
 import code.views.system.ViewDefinition
 import code.webhook.AccountWebhook
@@ -5731,7 +5732,7 @@ trait APIMethods310 {
     lazy val getWebUiProps: OBPEndpoint = {
       case "management" :: "webui_props":: Nil JsonGet req => {
         cc => implicit val ec = EndpointContext(Some(cc))
-          val active = S.param("active").getOrElse("false")
+          val active = ObpS.param("active").getOrElse("false")
           for {
             (Full(u), callContext) <- authenticatedAccess(cc)
             invalidMsg = s"""$InvalidFilterParameterFormat `active` must be a boolean, but current `active` value is: ${active} """

obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala

@@ -4,7 +4,6 @@ import java.net.URLEncoder
 import java.text.SimpleDateFormat
 import java.util
 import java.util.{Calendar, Date}
-
 import code.DynamicData.{DynamicData, DynamicDataProvider}
 import code.DynamicEndpoint.DynamicEndpointSwagger
 import code.accountattribute.AccountAttributeX
@@ -44,7 +43,7 @@ import code.api.v4_0_0.JSONFactory400._
 import code.api.dynamic.endpoint.helper._
 import code.api.dynamic.endpoint.helper.practise.PractiseEndpoint
 import code.api.dynamic.entity.helper.{DynamicEntityHelper, DynamicEntityInfo}
-import code.api.util.FutureUtil.{EndpointContext}
+import code.api.util.FutureUtil.EndpointContext
 import code.api.v5_0_0.OBPAPI5_0_0
 import code.api.{ChargePolicy, Constant, JsonResponseException}
 import code.apicollection.MappedApiCollectionsProvider
@@ -73,7 +72,7 @@ import code.transactionrequests.TransactionRequests.TransactionRequestTypes.{app
 import code.usercustomerlinks.UserCustomerLink
 import code.userlocks.UserLocksProvider
 import code.users.Users
-import code.util.Helper.booleanToFuture
+import code.util.Helper.{ObpS, booleanToFuture}
 import code.util.{Helper, JsonSchemaUtil}
 import code.validation.JsonValidation
 import code.views.Views
@@ -11762,8 +11761,8 @@ trait APIMethods400 {
     lazy val getAtms : OBPEndpoint = {
       case "banks" :: BankId(bankId) :: "atms" :: Nil JsonGet _ => {
         cc => implicit val ec = EndpointContext(Some(cc))
-          val limit = S.param("limit")
-          val offset = S.param("offset")
+          val limit = ObpS.param("limit")
+          val offset = ObpS.param("offset")
           for {
             (_, callContext) <- getAtmsIsPublic match {
               case false => authenticatedAccess(cc)