Merge pull request #2382 from hongwei1/develop

refactor/revert the migration function back

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -144,7 +144,7 @@ import net.liftweb.http.LiftRules.DispatchPF
 import net.liftweb.http._
 import net.liftweb.http.provider.HTTPCookie
 import net.liftweb.json.Extraction
-import net.liftweb.mapper._
+import net.liftweb.mapper.{DefaultConnectionIdentifier=>_, _}
 import net.liftweb.sitemap.Loc._
 import net.liftweb.sitemap._
 import net.liftweb.util.Helpers._
@@ -255,22 +255,22 @@ class Boot extends MdcLoggable {
 
     schemifyAll()
 
-    logger.info("Mapper database info: " + Migration.DbFunction.mapperDatabaseInfo(APIUtil.vendor))
-
-    //    DbFunction.tableExists(ResourceUser, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
-    //      case true => // DB already exist
-    //        // Migration Scripts are used to update the model of OBP-API DB to a latest version.
-    //        // Please note that migration scripts are executed before Lift Mapper Schemifier
-    ////        Migration.database.executeScripts(startedBeforeSchemifier = true)
-    //        logger.info("The Mapper database already exits. The scripts are executed BEFORE Lift Mapper Schemifier.")
-    //      case false => // DB is still not created. The scripts will be executed after Lift Mapper Schemifier
-    //        logger.info("The Mapper database is still not created. The scripts are going to be executed AFTER Lift Mapper Schemifier.")
-    //    }
+    logger.info("Mapper database info: " + Migration.DbFunction.mapperDatabaseInfo)
+
+    DbFunction.tableExists(ResourceUser) match {
+      case true => // DB already exist
+        // Migration Scripts are used to update the model of OBP-API DB to a latest version.
+        // Please note that migration scripts are executed before Lift Mapper Schemifier
+        Migration.database.executeScripts(startedBeforeSchemifier = true)
+        logger.info("The Mapper database already exits. The scripts are executed BEFORE Lift Mapper Schemifier.")
+      case false => // DB is still not created. The scripts will be executed after Lift Mapper Schemifier
+        logger.info("The Mapper database is still not created. The scripts are going to be executed AFTER Lift Mapper Schemifier.")
+    }
 
     // Migration Scripts are used to update the model of OBP-API DB to a latest version.
 
     // Please note that migration scripts are executed after Lift Mapper Schemifier
-    //Migration.database.executeScripts(startedBeforeSchemifier = false)
+    Migration.database.executeScripts(startedBeforeSchemifier = false)
 
     if (APIUtil.getPropsAsBoolValue("create_system_views_at_boot", true)) {
       // Create system views
@@ -285,18 +285,6 @@ class Boot extends MdcLoggable {
         Views.views.vend.getOrCreateSystemView(SYSTEM_FIREHOSE_VIEW_ID).isDefined
       else Empty.isDefined
 
-      val comment: String =
-        s"""
-           |System view ${SYSTEM_OWNER_VIEW_ID} exists/created at the instance: ${owner}
-           |System view ${SYSTEM_AUDITOR_VIEW_ID} exists/created at the instance: ${auditor}
-           |System view ${SYSTEM_ACCOUNTANT_VIEW_ID} exists/created at the instance: ${accountant}
-           |System view ${SYSTEM_FIREHOSE_VIEW_ID} exists/created at the instance: ${accountFirehose}
-           |System view ${SYSTEM_STANDARD_VIEW_ID} exists/created at the instance: ${standard}
-           |System view ${SYSTEM_STAGE_ONE_VIEW_ID} exists/created at the instance: ${stageOne}
-           |System view ${SYSTEM_MANAGE_CUSTOM_VIEWS_VIEW_ID} exists/created at the instance: ${manageCustomViews}
-           |""".stripMargin
-      logger.info(comment)
-
       APIUtil.getPropsValue("additional_system_views") match {
         case Full(value) =>
           val viewSetUKOpenBanking = value.split(",").map(_.trim).toList
@@ -313,8 +301,6 @@ class Boot extends MdcLoggable {
             if viewsUKOpenBanking.exists(_ == systemView)
           } {
             Views.views.vend.getOrCreateSystemView(systemView)
-            val comment = s"System view ${systemView} exists/created at the instance"
-            logger.info(comment)
           }
         case _ => // Do nothing
       }

obp-api/src/main/scala/code/api/util/migration/Migration.scala

@@ -2,11 +2,10 @@ package code.api.util.migration
 
 import bootstrap.liftweb.CustomDBVendor
 
-import java.sql.{ResultSet, SQLException}
+import java.sql.{Connection, ResultSet, SQLException}
 import java.text.SimpleDateFormat
 import java.util.Date
 import code.api.util.APIUtil.{getPropsAsBoolValue, getPropsValue}
-import code.api.util.ErrorMessages.DatabaseConnectionClosedError
 import code.api.util.{APIUtil, ApiPropsWithAlias}
 import code.api.v4_0_0.DatabaseInfoJson
 import code.consumer.Consumers
@@ -17,7 +16,7 @@ import code.util.Helper.MdcLoggable
 import com.github.dwickern.macros.NameOf.nameOf
 import com.zaxxer.hikari.pool.ProxyConnection
 import net.liftweb.mapper.Schemifier.getDefaultSchemaName
-import net.liftweb.mapper.{BaseMetaMapper, DB, SuperConnection}
+import net.liftweb.mapper.{BaseMetaMapper, DB, DefaultConnectionIdentifier, SuperConnection}
 
 import scala.collection.immutable
 import scala.collection.mutable.HashMap
@@ -492,55 +491,58 @@ object Migration extends MdcLoggable {
       * This function is copied from the module "net.liftweb.mapper.Schemifier".
       * The purpose is to provide answer does a table exist at a database instance.
       * For instance migration scripts needs to differentiate update of an instance from build a new one from scratch.
+     *  note: 07.05.2024 now. we get the connection from HikariDatasource.ds instead of Liftweb.
       */
-    def tableExists (table: BaseMetaMapper, connection: SuperConnection, actualTableNames: HashMap[String, String] = new HashMap[String, String]()): Boolean = {
-      val md = connection.getMetaData
-      using(md.getTables(null, getDefaultSchemaName(connection), null, null)){ rs =>
-        def hasTable(rs: ResultSet): Boolean =
-          if (!rs.next) false
-          else rs.getString(3) match {
-            case s if s.toLowerCase == table._dbTableNameLC.toLowerCase => actualTableNames(table._dbTableNameLC) = s; true
-            case _ => hasTable(rs)
+    def tableExists (table: BaseMetaMapper, actualTableNames: HashMap[String, String] = new HashMap[String, String]()): Boolean = {
+      DB.use(net.liftweb.util.DefaultConnectionIdentifier) {
+        conn =>
+          val md = conn.getMetaData
+          val schema =  getDefaultSchemaName(conn)
+
+          using(md.getTables(null, schema, null, null)){ rs =>
+            def hasTable(rs: ResultSet): Boolean =
+              if (!rs.next) false
+              else rs.getString(3) match {
+                case s if s.toLowerCase == table._dbTableNameLC.toLowerCase => actualTableNames(table._dbTableNameLC) = s; true
+                case _ => hasTable(rs)
+              }
+
+            hasTable(rs)
           }
-
-        hasTable(rs)
       }
     }
     /**
       * The purpose is to provide answer does a procedure exist at a database instance.
       */
-    def procedureExists(name: String, connection: SuperConnection): Boolean = {
-      val md = connection.getMetaData
-      using(md.getProcedures(null, getDefaultSchemaName(connection), null)){ rs =>
-        def hasProcedure(rs: ResultSet): Boolean =
-          if (!rs.next) false
-          else rs.getString(3) match {
-            case s if s.toLowerCase == name => true
-            case _ => hasProcedure(rs)
+    def procedureExists(name: String): Boolean = {
+      DB.use(net.liftweb.util.DefaultConnectionIdentifier) {
+        conn =>
+          val md = conn.getMetaData
+          val schema = getDefaultSchemaName(conn)
+          using(md.getProcedures(null, schema, null)){ rs =>
+            def hasProcedure(rs: ResultSet): Boolean =
+              if (!rs.next) false
+              else rs.getString(3) match {
+                case s if s.toLowerCase == name => true
+                case _ => hasProcedure(rs)
+              }
+            hasProcedure(rs)
           }
-        hasProcedure(rs)
       }
     }
 
 
     /**
       * The purpose is to provide info about the database in mapper mode.
       */
-    def mapperDatabaseInfo(vendor: CustomDBVendor): DatabaseInfoJson = {
-      val connection = vendor.createOne.openOrThrowException(DatabaseConnectionClosedError)
-      try {
-        val md = connection.getMetaData
-        val productName = md.getDatabaseProductName()
-        val productVersion = md.getDatabaseProductVersion()
-        DatabaseInfoJson(product_name = productName, product_version = productVersion)
-      } finally {
-        try {
-          connection.asInstanceOf[ProxyConnection].close()
-        } catch {
-          case t: Throwable => logger.error(s"mapperDatabaseInfo.close connection throw exception, detail is: $t")
-        }
+    def mapperDatabaseInfo: DatabaseInfoJson = {
+      DB.use(net.liftweb.util.DefaultConnectionIdentifier) {
+        conn =>
+          val md = conn.getMetaData
+          val productName = md.getDatabaseProductName()
+          val productVersion = md.getDatabaseProductVersion()
+          DatabaseInfoJson(product_name = productName, product_version = productVersion)
       }
-
     }
 
     /**
@@ -555,16 +557,19 @@ object Migration extends MdcLoggable {
       *
       * @return SQL command.
       */
-    def maybeWrite(performWrite: Boolean, logFunc: (=> AnyRef) => Unit, connection: SuperConnection) (makeSql: () => String) : String ={
-      val ct = makeSql()
-      logger.trace("maybeWrite DDL: "+ct)
-      if (performWrite) {
-        logFunc(ct)
-        val st = connection.createStatement
-        st.execute(ct)
-        st.close
-      }
-      ct
+    def maybeWrite(performWrite: Boolean, logFunc: (=> AnyRef) => Unit) (makeSql: () => String) : String ={
+      DB.use(net.liftweb.util.DefaultConnectionIdentifier) {
+        conn =>
+          val ct = makeSql()
+          logger.trace("maybeWrite DDL: " + ct)
+          if (performWrite) {
+            logFunc(ct)
+            val st = conn.createStatement
+            st.execute(ct)
+            st.close
+          }
+          ct
+      }
     }
 
     /**

obp-api/src/main/scala/code/api/util/migration/MigrationInfoOfAccoutHolders.scala

@@ -18,7 +18,7 @@ object BankAccountHoldersAndOwnerViewAccess {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def saveInfoBankAccountHoldersAndOwnerViewAccessInfo(name: String): Boolean = {
-    DbFunction.tableExists(MapperAccountHolders, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(MapperAccountHolders) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit

obp-api/src/main/scala/code/api/util/migration/MigrationOfAccountAccess.scala

@@ -9,7 +9,7 @@ import net.liftweb.util.DefaultConnectionIdentifier
 
 object TableAccountAccess {
   def populate(name: String): Boolean = {
-    DbFunction.tableExists(ViewPrivileges, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(ViewPrivileges) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit

obp-api/src/main/scala/code/api/util/migration/MigrationOfAccountAccessAddedConsumerId.scala

@@ -18,14 +18,14 @@ object MigrationOfAccountAccessAddedConsumerId {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def addAccountAccessConsumerId(name: String): Boolean = {
-    DbFunction.tableExists(AccountAccess, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(AccountAccess) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
         var isSuccessful = false
 
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             APIUtil.getPropsValue("db.driver") match    {
               case Full(value) if value.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") =>
                 () =>

obp-api/src/main/scala/code/api/util/migration/MigrationOfAccountRoutings.scala

@@ -17,7 +17,7 @@ object MigrationOfAccountRoutings {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def populate(name: String): Boolean = {
-    DbFunction.tableExists(BankAccountRouting, (DB.use(DefaultConnectionIdentifier) { conn => conn })) match {
+    DbFunction.tableExists(BankAccountRouting) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit

obp-api/src/main/scala/code/api/util/migration/MigrationOfAuthUser.scala

@@ -17,14 +17,14 @@ object MigrationOfAuthUser {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def alterColumnUsernameProviderEmailFirstnameAndLastname(name: String): Boolean = {
-    DbFunction.tableExists(AuthUser, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(AuthUser) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
         var isSuccessful = false
 
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             APIUtil.getPropsValue("db.driver") match    {
               case Full(value) if value.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") =>
                 () =>
@@ -70,14 +70,14 @@ object MigrationOfAuthUser {
   }
 
   def dropIndexAtColumnUsername(name: String): Boolean = {
-    DbFunction.tableExists(AuthUser, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(AuthUser) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
         var isSuccessful = false
 
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             APIUtil.getPropsValue("db.driver") match {
               case _ =>
                 () =>

obp-api/src/main/scala/code/api/util/migration/MigrationOfConsentAuthContextDropIndex.scala

@@ -38,14 +38,14 @@ object MigrationOfConsentAuthContextDropIndex {
   }
 
   def dropUniqueIndex(name: String): Boolean = {
-    DbFunction.tableExists(MappedConsentAuthContext, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(MappedConsentAuthContext) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
         var isSuccessful = false
 
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             APIUtil.getPropsValue("db.driver") match    {
               case _ =>
                 () => "DROP INDEX IF EXISTS consentauthcontext_consentid_key_c;"

obp-api/src/main/scala/code/api/util/migration/MigrationOfConsumer.scala

@@ -16,7 +16,7 @@ object MigrationOfConsumer {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def populateNamAndAppType(name: String): Boolean = {
-    DbFunction.tableExists(Consumer, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(Consumer) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
@@ -62,7 +62,7 @@ object MigrationOfConsumer {
     }
   }  
   def populateAzpAndSub(name: String): Boolean = {
-    DbFunction.tableExists(Consumer, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(Consumer) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit

obp-api/src/main/scala/code/api/util/migration/MigrationOfConsumerRateLimiting.scala

@@ -19,7 +19,7 @@ object TableRateLmiting {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def populate(name: String): Boolean = {
-    DbFunction.tableExists(RateLimiting, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(RateLimiting) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit

obp-api/src/main/scala/code/api/util/migration/MigrationOfCustomerAttributes.scala

@@ -18,14 +18,14 @@ object MigrationOfCustomerAttributes {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def alterColumnValue(name: String): Boolean = {
-    DbFunction.tableExists(MappedCustomerAttribute, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(MappedCustomerAttribute) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
         var isSuccessful = false
 
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             APIUtil.getPropsValue("db.driver") match    {
               case Full(value) if value.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") =>
                 () => "ALTER TABLE mappedcustomerattribute ALTER COLUMN mvalue varchar(2000);"
@@ -57,7 +57,7 @@ object MigrationOfCustomerAttributes {
     }
   }  
   def populateAzpAndSub(name: String): Boolean = {
-    DbFunction.tableExists(Consumer, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(Consumer) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit

obp-api/src/main/scala/code/api/util/migration/MigrationOfFastFireHoseMaterializedView.scala

@@ -17,7 +17,7 @@ object MigrationOfFastFireHoseMaterializedView {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def addFastFireHoseMaterializedView(name: String): Boolean = {
-    DbFunction.tableExists(ProductFee, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(ProductFee) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
@@ -75,7 +75,7 @@ object MigrationOfFastFireHoseMaterializedView {
              |                   ON (mappedbankaccount.bank = mapperaccountholders.accountbankpermalink and mappedbankaccount.theaccountid = mapperaccountholders.accountpermalink);
              |""".stripMargin
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             APIUtil.getPropsValue("db.driver") openOr("org.h2.Driver") match {
               case value if value.contains("org.h2.Driver") =>
                 () => migrationSql(false)//Note: H2 database, do not support the MATERIALIZED view

obp-api/src/main/scala/code/api/util/migration/MigrationOfFastFireHoseView.scala

@@ -16,14 +16,14 @@ object MigrationOfFastFireHoseView {
   val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
 
   def addFastFireHoseView(name: String): Boolean = {
-    DbFunction.tableExists(ProductFee, (DB.use(DefaultConnectionIdentifier){ conn => conn})) match {
+    DbFunction.tableExists(ProductFee) match {
       case true =>
         val startDate = System.currentTimeMillis()
         val commitId: String = APIUtil.gitCommit
         var isSuccessful = false
 
         val executedSql =
-          DbFunction.maybeWrite(true, Schemifier.infoF _, DB.use(DefaultConnectionIdentifier){ conn => conn}) {
+          DbFunction.maybeWrite(true, Schemifier.infoF _) {
             () =>
               """
                 |CREATE VIEW v_fast_firehose_accounts AS select