Fixes/fix db schema (#90)

* Change from VARCHAR to TEXT to work with mysql * Delete empty block * Revert empty block deletion * Migrate genepi-db classes * Fix empty SQL block execution * Fix issue with non-unique schemas
genepi · Oct 3, 2023 · c05ed33 · c05ed33
1 parent 4f9ba91
commit c05ed33
Show file tree

Hide file tree

Showing 68 changed files with 1,529 additions and 124 deletions.
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
@@ -1,5 +1,8 @@
 eclipse.preferences.version=1
 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
 org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=disabled
 org.eclipse.jdt.core.compiler.source=1.8
diff --git a/pom.xml b/pom.xml
@@ -94,19 +94,17 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
-
 		<dependency>
-			<groupId>genepi</groupId>
-			<artifactId>genepi-db</artifactId>
-			<version>1.2.1</version>
-			<exclusions>
-				<exclusion>
-					<groupId>log4j</groupId>
-					<artifactId>log4j</artifactId>
-				</exclusion>
-			</exclusions>
-		</dependency>
-
+				<groupId>commons-dbutils</groupId>
+				<artifactId>commons-dbutils</artifactId>
+				<version>1.7</version>
+			</dependency>
+
+			<dependency>
+				<groupId>commons-dbcp</groupId>
+				<artifactId>commons-dbcp</artifactId>
+				<version>1.4</version>
+			</dependency>
 		<dependency>
 			<groupId>genepi</groupId>
 			<artifactId>genepi-hadoop</artifactId>

diff --git a/src/main/java/cloudgene/mapred/Main.java b/src/main/java/cloudgene/mapred/Main.java
@@ -21,17 +21,17 @@
 import com.esotericsoftware.yamlbeans.YamlReader;
 
 import cloudgene.mapred.database.updates.BcryptHashUpdate;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.DatabaseConnector;
 import cloudgene.mapred.database.util.DatabaseConnectorFactory;
+import cloudgene.mapred.database.util.DatabaseUpdater;
 import cloudgene.mapred.database.util.Fixtures;
 import cloudgene.mapred.jobs.PersistentWorkflowEngine;
 import cloudgene.mapred.jobs.WorkflowEngine;
 import cloudgene.mapred.plugins.PluginManager;
 import cloudgene.mapred.util.BuildUtil;
 import cloudgene.mapred.util.Config;
 import cloudgene.mapred.util.Settings;
-import genepi.db.Database;
-import genepi.db.DatabaseConnector;
-import genepi.db.DatabaseUpdater;
 import genepi.io.FileUtil;
 
 public class Main {

diff --git a/src/main/java/cloudgene/mapred/WebApp.java b/src/main/java/cloudgene/mapred/WebApp.java
@@ -73,15 +73,14 @@
 import cloudgene.mapred.api.v2.users.UpdatePassword;
 import cloudgene.mapred.api.v2.users.UserProfile;
 import cloudgene.mapred.api.v2.users.VerifyApiToken;
-import cloudgene.mapred.apps.ApplicationRepository;
 import cloudgene.mapred.database.TemplateDao;
+import cloudgene.mapred.database.util.Database;
 import cloudgene.mapred.jobs.WorkflowEngine;
 import cloudgene.mapred.representations.CustomStatusService;
 import cloudgene.mapred.resources.Admin;
 import cloudgene.mapred.resources.Index;
 import cloudgene.mapred.resources.Start;
 import cloudgene.mapred.util.Settings;
-import genepi.db.Database;
 
 public class WebApp extends Application {
 

diff --git a/src/main/java/cloudgene/mapred/WebServer.java b/src/main/java/cloudgene/mapred/WebServer.java
@@ -8,11 +8,10 @@
 import org.restlet.data.Protocol;
 import org.restlet.routing.VirtualHost;
 
-import cloudgene.mapred.apps.Application;
 import cloudgene.mapred.cron.CronJobScheduler;
+import cloudgene.mapred.database.util.Database;
 import cloudgene.mapred.jobs.WorkflowEngine;
 import cloudgene.mapred.util.Settings;
-import genepi.db.Database;
 
 public class WebServer extends Component {
 

diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/NextflowWebLog.java b/src/main/java/cloudgene/mapred/api/v2/jobs/NextflowWebLog.java
@@ -1,7 +1,5 @@
 package cloudgene.mapred.api.v2.jobs;
 
-import java.io.IOException;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.restlet.representation.Representation;

diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java
@@ -1,8 +1,6 @@
 package cloudgene.mapred.apps;
 
 import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -18,15 +16,13 @@
 
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.esotericsoftware.yamlbeans.YamlReader;
-import com.esotericsoftware.yamlbeans.YamlWriter;
 
 import cloudgene.mapred.core.User;
+import cloudgene.mapred.database.util.DatabaseUpdater;
 import cloudgene.mapred.util.GitHubException;
 import cloudgene.mapred.util.GitHubUtil;
 import cloudgene.mapred.util.GitHubUtil.Repository;
 import cloudgene.mapred.wdl.WdlApp;
-import genepi.db.DatabaseUpdater;
 import genepi.hadoop.S3Util;
 import genepi.io.FileUtil;
 import net.lingala.zip4j.ZipFile;

diff --git a/src/main/java/cloudgene/mapred/core/ApiTokenVerifier.java b/src/main/java/cloudgene/mapred/core/ApiTokenVerifier.java
@@ -10,7 +10,7 @@
 import com.nimbusds.jose.crypto.MACVerifier;
 
 import cloudgene.mapred.database.UserDao;
-import genepi.db.Database;
+import cloudgene.mapred.database.util.Database;
 
 public class ApiTokenVerifier {
 

diff --git a/src/main/java/cloudgene/mapred/core/JWTUtil.java b/src/main/java/cloudgene/mapred/core/JWTUtil.java
@@ -10,7 +10,7 @@
 import org.restlet.util.Series;
 
 import cloudgene.mapred.database.UserDao;
-import genepi.db.Database;
+import cloudgene.mapred.database.util.Database;
 
 public class JWTUtil {
 

diff --git a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java
@@ -8,13 +8,13 @@
 
 import cloudgene.mapred.WebApp;
 import cloudgene.mapred.database.JobDao;
+import cloudgene.mapred.database.util.Database;
 import cloudgene.mapred.jobs.AbstractJob;
 import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory;
 import cloudgene.mapred.util.MailUtil;
 import cloudgene.mapred.util.Settings;
 import cloudgene.mapred.util.Template;
 import cloudgene.sdk.internal.IExternalWorkspace;
-import genepi.db.Database;
 import genepi.hadoop.HdfsUtil;
 import genepi.io.FileUtil;
 

diff --git a/src/main/java/cloudgene/mapred/cron/RetireJob.java b/src/main/java/cloudgene/mapred/cron/RetireJob.java
@@ -6,8 +6,8 @@
 import org.quartz.JobExecutionException;
 
 import cloudgene.mapred.WebApp;
+import cloudgene.mapred.database.util.Database;
 import cloudgene.mapred.util.Settings;
-import genepi.db.Database;
 
 public class RetireJob implements Job {
 

diff --git a/src/main/java/cloudgene/mapred/cron/StatisticsJob.java b/src/main/java/cloudgene/mapred/cron/StatisticsJob.java
@@ -12,9 +12,9 @@
 import cloudgene.mapred.core.User;
 import cloudgene.mapred.database.CounterHistoryDao;
 import cloudgene.mapred.database.UserDao;
+import cloudgene.mapred.database.util.Database;
 import cloudgene.mapred.jobs.AbstractJob;
 import cloudgene.mapred.jobs.WorkflowEngine;
-import genepi.db.Database;
 
 public class StatisticsJob implements Job {
 

diff --git a/src/main/java/cloudgene/mapred/database/CounterDao.java b/src/main/java/cloudgene/mapred/database/CounterDao.java
@@ -8,10 +8,10 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.AbstractJob;
-import genepi.db.Database;
-import genepi.db.IRowMapMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class CounterDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java b/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java
@@ -14,8 +14,8 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import genepi.db.Database;
-import genepi.db.JdbcDataAccessObject;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 
 public class CounterHistoryDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/DownloadDao.java b/src/main/java/cloudgene/mapred/database/DownloadDao.java
@@ -8,11 +8,11 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.CloudgeneParameterOutput;
 import cloudgene.mapred.jobs.Download;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class DownloadDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java
@@ -10,14 +10,15 @@
 
 import cloudgene.mapred.core.User;
 import cloudgene.mapred.database.UserDao.UserMapper;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
+import cloudgene.mapred.database.util.JdbcDataAccessObject.IntegerMapper;
 import cloudgene.mapred.jobs.AbstractJob;
 import cloudgene.mapred.jobs.CloudgeneJob;
 import cloudgene.mapred.jobs.CloudgeneParameterInput;
 import cloudgene.mapred.jobs.CloudgeneParameterOutput;
 import cloudgene.mapred.jobs.CloudgeneStep;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class JobDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/MessageDao.java b/src/main/java/cloudgene/mapred/database/MessageDao.java
@@ -8,11 +8,11 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.CloudgeneStep;
 import cloudgene.mapred.jobs.Message;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class MessageDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java
@@ -8,15 +8,15 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.AbstractJob;
 import cloudgene.mapred.jobs.CloudgeneParameterInput;
 import cloudgene.mapred.jobs.CloudgeneParameterOutput;
 import cloudgene.mapred.jobs.Download;
 import cloudgene.mapred.wdl.WdlParameterInputType;
 import cloudgene.mapred.wdl.WdlParameterOutputType;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class ParameterDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/StepDao.java b/src/main/java/cloudgene/mapred/database/StepDao.java
@@ -8,13 +8,13 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.CloudgeneJob;
 import cloudgene.mapred.jobs.CloudgeneStep;
 import cloudgene.mapred.jobs.Message;
 import cloudgene.mapred.steps.EmptyStep;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class StepDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/TemplateDao.java b/src/main/java/cloudgene/mapred/database/TemplateDao.java
@@ -8,10 +8,10 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.util.Template;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class TemplateDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/UserDao.java b/src/main/java/cloudgene/mapred/database/UserDao.java
@@ -9,10 +9,10 @@
 import org.apache.commons.logging.LogFactory;
 
 import cloudgene.mapred.core.User;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.util.PublicUser;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class UserDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java b/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java
@@ -8,8 +8,8 @@
 
 import cloudgene.mapred.core.User;
 import cloudgene.mapred.database.UserDao;
-import genepi.db.Database;
-import genepi.db.IUpdateListener;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IUpdateListener;
 
 public class BcryptHashUpdate implements IUpdateListener {