Configure database connection via db.connection_string param

Needed for flexible configuration of connection parameters required to run MySQL8.

.github/workflows/integration-test.yml

@@ -42,10 +42,12 @@ jobs:
         working-directory: ./cbioportal-docker-compose
         run: |
           cd ./data && ./init.sh && rm -rf ./studies/* && cd ../config && \
-          cat $PORTAL_SOURCE_DIR/portal/target/portal/WEB-INF/classes/portal.properties | \
-              sed 's/db.host=.*/db.host=cbioportal-database:3306/g' | \
-              sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/|g' \
-              > portal.properties
+          cat $PORTAL_SOURCE_DIR/portal/target/classes/portal.properties | \
+              sed 's|db.host=.*||' | \
+              sed 's|db.portal_db_name=.*||' | \
+              sed 's|db.use_ssl=.*||' | \
+              sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false\&allowPublicKeyRetrieval=true|' \
+              > portal.properties && more portal.properties
       - name: 'Start cbioportal-docker-compose'
         working-directory: ./cbioportal-docker-compose
         run: |

core/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java

@@ -3,28 +3,44 @@
 import org.apache.commons.dbcp2.BasicDataSource;
 import org.mskcc.cbio.portal.util.DatabaseProperties;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.Assert;
 
 /**
  * Data source that self-initializes based on cBioPortal configuration.
  */
 public class JdbcDataSource extends BasicDataSource {
+
     public JdbcDataSource () {
         DatabaseProperties dbProperties = DatabaseProperties.getInstance();
+
         String host = dbProperties.getDbHost();
         String userName = dbProperties.getDbUser();
         String password = dbProperties.getDbPassword();
         String mysqlDriverClassName = dbProperties.getDbDriverClassName();
         String database = dbProperties.getDbName();
-        String useSSL = (!StringUtils.isBlank(dbProperties.getDbUseSSL())) ? dbProperties.getDbUseSSL() : "false";
         String enablePooling = (!StringUtils.isBlank(dbProperties.getDbEnablePooling())) ? dbProperties.getDbEnablePooling(): "false";
-        String url ="jdbc:mysql://" + host + "/" + database +
-                        "?user=" + userName + "&password=" + password +
-                        "&zeroDateTimeBehavior=convertToNull&useSSL=" + useSSL;
+        String connectionURL = dbProperties.getConnectionURL();
+
+        Assert.isTrue(
+            !defined(host) && !defined(database) && !defined(dbProperties.getDbUseSSL()),
+            "\n----------------------------------------------------------------------------------------------------------------" +
+                "-- Connection error:\n" +
+                "-- You try to connect to the database using the deprecated 'db.host', 'db.portal_db_name' and 'db.use_ssl' properties.\n" +
+                "-- Please remove these properties and use the 'db.connection_string' property instead. See https://docs.cbioportal.org/deployment/customization/portal.properties-reference/\n" +
+                "-- for assistance on building a valid connection string.\n" +
+                "----------------------------------------------------------------------------------------------------------------\n"
+        );
+
+        Assert.hasText(userName, errorMessage("username", "db.user"));
+        Assert.hasText(password, errorMessage("password", "db.password"));
+        Assert.hasText(mysqlDriverClassName, errorMessage("driver class name", "db.driver"));
+
+        this.setUrl(connectionURL);
+
         //  Set up poolable data source
         this.setDriverClassName(mysqlDriverClassName);
         this.setUsername(userName);
         this.setPassword(password);
-        this.setUrl(url);
         // Disable this to avoid caching statements
         this.setPoolPreparedStatements(Boolean.valueOf(enablePooling));
         // these are the values cbioportal has been using in their production
@@ -37,4 +53,12 @@ public JdbcDataSource () {
         this.setValidationQuery("SELECT 1");
         this.setJmxName("org.cbioportal:DataSource=" + database);
     }
+
+    private String errorMessage(String displayName, String propertyName) {
+        return String.format("No %s provided for database connection. Please set '%s' in portal.properties.", displayName, propertyName);
+    }
+
+    private boolean defined(String property) {
+        return property != null && !property.isEmpty();
+    }
 }

core/src/main/java/org/mskcc/cbio/portal/util/DatabaseProperties.java

@@ -47,6 +47,7 @@ public class DatabaseProperties {
     private String dbDriverClassName;
     private String dbUseSSL;
     private String dbEnablePooling;
+    private String connectionURL;
 
     // No production keys stored in filesystem or code: digest the key; put it in properties; load it into dbms on startup
     private static DatabaseProperties dbProperties;
@@ -63,6 +64,7 @@ public static DatabaseProperties getInstance() {
             dbProperties.setDbDriverClassName(GlobalProperties.getProperty("db.driver"));
             dbProperties.setDbUseSSL(GlobalProperties.getProperty("db.use_ssl"));
             dbProperties.setDbEnablePooling(GlobalProperties.getProperty("db.enable_pooling"));
+            dbProperties.setConnectionURL(GlobalProperties.getProperty("db.connection_string"));
         }
         return dbProperties;
     }
@@ -134,4 +136,12 @@ public void setDbEnablePooling(String dbEnablePooling) {
         this.dbEnablePooling = dbEnablePooling;
     }
 
+    public String getConnectionURL() {
+        return connectionURL;
+    }
+
+    public void setConnectionURL(String connectionURL) {
+        this.connectionURL = connectionURL;
+    }
+
 }

core/src/main/java/org/mskcc/cbio/portal/util/SpringUtil.java

@@ -38,33 +38,28 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.context.support.GenericXmlApplicationContext;
 import org.springframework.stereotype.Component;
 
 @Component
-public class SpringUtil
-{
+public class SpringUtil {
     private static final Logger log = LoggerFactory.getLogger(SpringUtil.class);
 
     private static AccessControl accessControl;
-    private static ApplicationContext context;
-    private static GenericXmlApplicationContext applicationContext;
+    private static ApplicationContext applicationContext;
 
     @Autowired
     public void setAccessControl(AccessControl accessControl) {
         log.debug("Setting access control");
         SpringUtil.accessControl = accessControl;
     }
 
-    public static AccessControl getAccessControl()
-    {
+    public static AccessControl getAccessControl() {
         return accessControl;
     }
 
-    public static synchronized void initDataSource()
-    {
-        if (SpringUtil.context == null) {
-            context = new ClassPathXmlApplicationContext("classpath:applicationContext-persistenceConnections.xml");
+    public static synchronized void initDataSource() {
+        if (SpringUtil.applicationContext == null) {
+            SpringUtil.applicationContext = new ClassPathXmlApplicationContext("classpath:applicationContext-persistenceConnections.xml");
         }
     }
 
@@ -74,7 +69,7 @@ public static synchronized void initDataSource()
      * @return the Spring Framework application context
      */
     public static ApplicationContext getApplicationContext() {
-        return context;
+        return applicationContext;
     }
 
     /**
@@ -84,7 +79,7 @@ public static ApplicationContext getApplicationContext() {
      * @param context
      */
     public static void setApplicationContext(ApplicationContext context) {
-        SpringUtil.context = context;
+        SpringUtil.applicationContext = context;
     }
 
     /**
@@ -93,8 +88,7 @@ public static void setApplicationContext(ApplicationContext context) {
      *
      * @param context
      */
-    public static synchronized void initDataSource(ApplicationContext context)
-    {
-        SpringUtil.context = context;
+    public static synchronized void initDataSource(ApplicationContext context) {
+        SpringUtil.applicationContext = context;
     }
 }

core/src/main/scripts/importer/cbioportalImporter.py

@@ -419,8 +419,7 @@ def usage():
                            '--command [%s] --study_directory <path to directory> '
                            '--meta_filename <path to metafile>'
                            '--data_filename <path to datafile>'
-                           '--study_ids <cancer study ids for remove-study command, comma separated>'
-                           '--properties-filename <path to properties file> ' % (COMMANDS)), file=OUTPUT_FILE)
+                           '--study_ids <cancer study ids for remove-study command, comma separated>' % (COMMANDS)), file=OUTPUT_FILE)
 
 def check_args(command):
     if command not in COMMANDS:

core/src/main/scripts/importer/cbioportal_common.py

@@ -11,7 +11,9 @@
 import logging.handlers
 from collections import OrderedDict
 from subprocess import Popen, PIPE, STDOUT
-
+from typing import Dict, Optional
+import dsnparse
+import MySQLdb
 
 # ------------------------------------------------------------------------------
 # globals
@@ -37,6 +39,14 @@
 ADD_CASE_LIST_CLASS = "org.mskcc.cbio.portal.scripts.AddCaseList"
 VERSION_UTIL_CLASS = "org.mskcc.cbio.portal.util.VersionUtil"
 
+PORTAL_PROPERTY_DATABASE_USER = 'db.user'
+PORTAL_PROPERTY_DATABASE_PW = 'db.password'
+PORTAL_PROPERTY_DATABASE_HOST = 'db.host'
+PORTAL_PROPERTY_DATABASE_NAME = 'db.portal_db_name'
+PORTAL_PROPERTY_DATABASE_URL = 'db.connection_string'
+PORTAL_PROPERTY_DATABASE_USESSL = 'db.use_ssl'
+REQUIRED_DATABASE_PROPERTIES = [PORTAL_PROPERTY_DATABASE_USER, PORTAL_PROPERTY_DATABASE_PW, PORTAL_PROPERTY_DATABASE_URL]
+
 # provides a key for data types to metafile specification dict.
 class MetaFileTypes(object):
     """how we differentiate between data types."""
@@ -1000,3 +1010,96 @@ def run_java(*args):
     elif process.returncode != 0:
         raise RuntimeError('Aborting due to error while executing step.')
     return ret
+
+
+def properties_error_message(display_name: str, property_name: str) -> str:
+    return f"No {display_name} provided for database connection. Please set '{property_name}' in portal.properties."
+
+
+class PortalProperties(object):
+    """ Properties object class, just has fields for db conn """
+
+    def __init__(self, database_user, database_pw, database_url):
+        self.database_user = database_user
+        self.database_pw = database_pw
+        self.database_url = database_url
+
+
+def get_database_properties(properties_filename: str) -> Optional[PortalProperties]:
+
+    properties = parse_properties_file(properties_filename)
+
+    missing_properties = []
+    for required_property in REQUIRED_DATABASE_PROPERTIES:
+        if required_property not in properties or len(properties[required_property]) == 0:
+            missing_properties.append(required_property)
+    if missing_properties:
+        print(
+            'Missing required properties : (%s)' % (', '.join(missing_properties)),
+            file=ERROR_FILE)
+        return None
+
+    if properties.get(PORTAL_PROPERTY_DATABASE_HOST) is not None \
+        or properties.get(PORTAL_PROPERTY_DATABASE_NAME) is not None \
+        or properties.get(PORTAL_PROPERTY_DATABASE_USESSL) is not None:
+        print("""
+            ----------------------------------------------------------------------------------------------------------------
+            -- Connection error:
+            -- You try to connect to the database using the deprecated 'db.host', 'db.portal_db_name' and 'db.use_ssl' properties.
+            -- Please remove these properties and use the 'db.connection_string' property instead. See https://docs.cbioportal.org/deployment/customization/portal.properties-reference/
+            -- for assistance on building a valid connection string.
+            ------------------------------------------------------------f---------------------------------------------------
+        """, file=ERROR_FILE)
+        return None
+
+    return PortalProperties(properties[PORTAL_PROPERTY_DATABASE_USER],
+                            properties[PORTAL_PROPERTY_DATABASE_PW],
+                            properties[PORTAL_PROPERTY_DATABASE_URL])
+
+
+def parse_properties_file(properties_filename: str) -> Dict[str, str]:
+
+    if not os.path.exists(properties_filename):
+        print('properties file %s cannot be found' % properties_filename, file=ERROR_FILE)
+        sys.exit(2)
+
+    properties = {}
+    with open(properties_filename, 'r') as properties_file:
+        for line in properties_file:
+            line = line.strip()
+            # skip line if its blank or a comment
+            if len(line) == 0 or line.startswith('#'):
+                continue
+            try:
+                name, value = line.split('=', maxsplit=1)
+            except ValueError:
+                print(
+                    'Skipping invalid entry in property file: %s' % line,
+                    file=ERROR_FILE)
+                continue
+            properties[name] = value.strip()
+    return properties
+
+
+def get_db_cursor(portal_properties: PortalProperties):
+
+    try:
+        url_elements = dsnparse.parse(portal_properties.database_url)
+        connection_kwargs = {
+            "host": url_elements.host,
+            "port": url_elements.port if url_elements.port is not None else 3306,
+            "db": url_elements.paths[0],
+            "user": portal_properties.database_user,
+            "passwd": portal_properties.database_pw,
+            "ssl": "useSSL" not in url_elements.query or url_elements.query["useSSL"] == "true"
+        }
+        connection = MySQLdb.connect(**connection_kwargs)
+    except MySQLdb.Error as exception:
+        print(exception, file=ERROR_FILE)
+        message = (
+            "--> Error connecting to server with URL"
+            + portal_properties.database_url)
+        print(message, file=ERROR_FILE)
+        raise ConnectionError(message) from exception
+    if connection is not None:
+        return connection, connection.cursor()