diff --git a/config/vufind/config.ini b/config/vufind/config.ini index db7f8190391..5b00ad3374f 100644 --- a/config/vufind/config.ini +++ b/config/vufind/config.ini @@ -817,7 +817,7 @@ url_shortener_key_type = md5 [Database] ; Connection string format is [platform]://[username]:[password]@[host]:[port]/[db] ; where: -; [platform/driver] = database platform (mysql, oci8 or pgsql) +; [platform/driver] = database platform (mysql, mariadb, oci8 or pgsql) ; [username] = username for connection ; [password] = password for connection (optional) ; [host] = host of database server diff --git a/import/index_java/src/org/vufind/index/DatabaseManager.java b/import/index_java/src/org/vufind/index/DatabaseManager.java index 4173c7d0293..d4bedd1520d 100644 --- a/import/index_java/src/org/vufind/index/DatabaseManager.java +++ b/import/index_java/src/org/vufind/index/DatabaseManager.java @@ -40,6 +40,8 @@ public class DatabaseManager // Shutdown flag: private boolean shuttingDown = false; + private UpdateDateTracker udt; + private static ThreadLocal managerCache = new ThreadLocal() { @@ -102,9 +104,11 @@ private void connectToDatabaseUsingStringConfig() throws Throwable // Parse key settings from the PHP-style DSN: String platform = "invalid"; - if (dsn.substring(0, 8).equals("mysql://")) { + if (dsn.startsWith("mysql://")) { platform = "mysql"; - } else if (dsn.substring(0, 8).equals("pgsql://")) { + } else if (dsn.startsWith("mariadb://")) { + platform = "mariadb"; + } else if (dsn.startsWith("pgsql://")) { platform = "postgresql"; } @@ -172,13 +176,37 @@ private void connectToDatabaseUsingParams(String platform, String host, String p prefix = "mysql"; String useSsl = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "use_ssl", false) ? "true" : "false"; extraParams = "?useSSL=" + useSsl; + // NOTE: useSSL is deprecated for both mysql and mariadb + // sslMode is supported since mysql Connector/J 8.0.13 and mariadb Connector/J 3.0.0 if (useSsl != "false") { String verifyCert = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "verify_server_certificate", false) ? "true" : "false"; extraParams += "&verifyServerCertificate=" + verifyCert; } + extraParams += "&rewriteBatchedStatements=true"; + } else if (platform.equals("mariadb")) { + classname = "org.mariadb.jdbc.Driver"; + prefix = "mariadb"; + boolean useSsl = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "use_ssl", false); + String sslMode; + if (useSsl) { + boolean verifyCert = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "verify_server_certificate", false); + if (verifyCert) { + sslMode = "verify-full"; + // NOTE: we could also use "verify-ca" here, it would be better to have an sslMode VuFind config + // (but note that mysql values are different) + } else { + sslMode = "trust"; + } + } else { + sslMode = "disable"; + } + extraParams = "?sslMode=" + sslMode; + // NOTE: rewriteBatchedStatements was removed from mariadb Connector/J in version 3.0.0. + // It was replaced by useBulkStmts, which defaults to true. } else if (platform.equals("pgsql") || platform.equals("postgresql")) { classname = "org.postgresql.Driver"; prefix = "postgresql"; + extraParams = "?reWriteBatchedInserts=true"; } Class.forName(classname).getDeclaredConstructor().newInstance(); @@ -194,6 +222,10 @@ private void connectToDatabaseUsingParams(String platform, String host, String p Runtime.getRuntime().addShutdownHook(new DatabaseManagerShutdownThread(this)); } + void setUpdateDateTracker(UpdateDateTracker udt) { + this.udt = udt; + } + private void disconnectFromDatabase() { if (vufindDatabase != null) { @@ -208,6 +240,10 @@ private void disconnectFromDatabase() public void shutdown() { + if (udt != null) { + // We can't use UpdateDateTracker.instance() in shutdown, so we have to use a previously-saved reference + udt.shutdown(); + } disconnectFromDatabase(); shuttingDown = true; } diff --git a/import/index_java/src/org/vufind/index/UpdateDateTracker.java b/import/index_java/src/org/vufind/index/UpdateDateTracker.java index 28f4750e966..7fa33b8e77d 100644 --- a/import/index_java/src/org/vufind/index/UpdateDateTracker.java +++ b/import/index_java/src/org/vufind/index/UpdateDateTracker.java @@ -22,11 +22,19 @@ import java.time.format.DateTimeFormatter; import java.time.LocalDateTime; +import org.apache.log4j.Logger; + /** * Class for managing record update dates. */ public class UpdateDateTracker { + private static Logger logger = Logger.getLogger(UpdateDateTracker.class); + + private static final int BATCH_SIZE = 100; + private int insertBatchCount = 0; + private int updateBatchCount = 0; + private Connection db; private String core; private String id; @@ -37,6 +45,10 @@ public class UpdateDateTracker private Timestamp lastRecordChange; private Timestamp deleted; + PreparedStatement insertSql; + PreparedStatement selectSql; + PreparedStatement updateSql; + private static ThreadLocal trackerCache = new ThreadLocal() { @@ -44,7 +56,10 @@ public class UpdateDateTracker protected UpdateDateTracker initialValue() { try { - return new UpdateDateTracker(DatabaseManager.instance().getConnection()); + DatabaseManager dbm = DatabaseManager.instance(); + UpdateDateTracker udt = new UpdateDateTracker(dbm.getConnection()); + dbm.setUpdateDateTracker(udt); + return udt; } catch (SQLException e) { throw new RuntimeException(e.getMessage()); } @@ -56,6 +71,25 @@ public static UpdateDateTracker instance() return trackerCache.get(); } + private void possiblyExecuteBatch(boolean update, PreparedStatement statement, boolean force) throws SQLException + { + int count = update ? updateBatchCount : insertBatchCount; + if (count >= BATCH_SIZE || (count > 0 && force)) { + try { + statement.executeBatch(); + db.commit(); + } catch (SQLException ex) { + logger.error("SQLException in possiblyExecuteBatch(): " + ex.getMessage()); + throw ex; + } + if (update) { + updateBatchCount = 0; + } else { + insertBatchCount = 0; + } + } + } + /* Private support method: create a row in the change_tracker table. */ private void createRow(Timestamp newRecordChange) throws SQLException @@ -65,46 +99,32 @@ private void createRow(Timestamp newRecordChange) throws SQLException lastRecordChange = newRecordChange; // Save new values to the database: - try ( - PreparedStatement insertSql = db.prepareStatement( - "INSERT INTO change_tracker(core, id, first_indexed, last_indexed, last_record_change) " + - "VALUES(?, ?, ?, ?, ?);" - ); - ) { - insertSql.setString(1, core); - insertSql.setString(2, id); - insertSql.setTimestamp(3, firstIndexed); - insertSql.setTimestamp(4, lastIndexed); - insertSql.setTimestamp(5, lastRecordChange); - insertSql.executeUpdate(); - } + insertSql.setString(1, core); + insertSql.setString(2, id); + insertSql.setTimestamp(3, firstIndexed); + insertSql.setTimestamp(4, lastIndexed); + insertSql.setTimestamp(5, lastRecordChange); + insertSql.addBatch(); + insertBatchCount++; + possiblyExecuteBatch(false, insertSql, false); } /* Private support method: read a row from the change_tracker table. */ private boolean readRow() throws SQLException { - try ( - PreparedStatement selectSql = db.prepareStatement( - "SELECT first_indexed, last_indexed, last_record_change, deleted " + - "FROM change_tracker WHERE core = ? AND id = ?;", - ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY - ) - ) { - selectSql.setString(1, core); - selectSql.setString(2, id); - try (ResultSet result = selectSql.executeQuery()) { - // No results? Return false: - if (!result.first()) { - return false; - } else { - // If we got this far, we have results -- load them into the object: - firstIndexed = result.getTimestamp(1); - lastIndexed = result.getTimestamp(2); - lastRecordChange = result.getTimestamp(3); - deleted = result.getTimestamp(4); - } + selectSql.setString(1, core); + selectSql.setString(2, id); + try (ResultSet result = selectSql.executeQuery()) { + // No results? Free resources and return false: + if (!result.first()) { + return false; } + // If we got this far, we have results -- load them into the object: + firstIndexed = result.getTimestamp(1); + lastIndexed = result.getTimestamp(2); + lastRecordChange = result.getTimestamp(3); + deleted = result.getTimestamp(4); } return true; } @@ -123,21 +143,15 @@ private void updateRow(Timestamp newRecordChange) throws SQLException lastRecordChange = newRecordChange; // Save new values to the database: - try ( - PreparedStatement updateSql = db.prepareStatement( - "UPDATE change_tracker " + - "SET first_indexed = ?, last_indexed = ?, last_record_change = ?, deleted = ? " + - "WHERE core = ? AND id = ?;" - ) - ) { - updateSql.setTimestamp(1, firstIndexed); - updateSql.setTimestamp(2, lastIndexed); - updateSql.setTimestamp(3, lastRecordChange); - updateSql.setNull(4, java.sql.Types.NULL); - updateSql.setString(5, core); - updateSql.setString(6, id); - updateSql.executeUpdate(); - } + updateSql.setTimestamp(1, firstIndexed); + updateSql.setTimestamp(2, lastIndexed); + updateSql.setTimestamp(3, lastRecordChange); + updateSql.setNull(4, java.sql.Types.NULL); + updateSql.setString(5, core); + updateSql.setString(6, id); + updateSql.addBatch(); + updateBatchCount++; + possiblyExecuteBatch(true, updateSql, false); } /* Constructor: @@ -145,6 +159,29 @@ private void updateRow(Timestamp newRecordChange) throws SQLException public UpdateDateTracker(Connection dbConnection) throws SQLException { db = dbConnection; + db.setAutoCommit(false); + insertSql = db.prepareStatement( + "INSERT INTO change_tracker(core, id, first_indexed, last_indexed, last_record_change) " + + "VALUES(?, ?, ?, ?, ?);"); + selectSql = db.prepareStatement( + "SELECT first_indexed, last_indexed, last_record_change, deleted " + + "FROM change_tracker WHERE core = ? AND id = ?;", + ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + updateSql = db.prepareStatement("UPDATE change_tracker " + + "SET first_indexed = ?, last_indexed = ?, last_record_change = ?, deleted = ? " + + "WHERE core = ? AND id = ?;"); + } + + void shutdown() { + try { + possiblyExecuteBatch(false, insertSql, true); + possiblyExecuteBatch(true, updateSql, true); + insertSql.close(); + selectSql.close(); + updateSql.close(); + } catch (SQLException ex) { + logger.error("SQLException in shutdown hook: " + ex.getMessage()); + } } /* Get the first indexed date (IMPORTANT: index() must be called before this method) diff --git a/import/lib_local/mariadb-java-client-3.5.1.jar b/import/lib_local/mariadb-java-client-3.5.1.jar new file mode 100644 index 00000000000..46f5e1f2fc0 Binary files /dev/null and b/import/lib_local/mariadb-java-client-3.5.1.jar differ diff --git a/module/VuFind/src/VuFind/Db/AdapterFactory.php b/module/VuFind/src/VuFind/Db/AdapterFactory.php index 339ca2d9e4a..5552d787483 100644 --- a/module/VuFind/src/VuFind/Db/AdapterFactory.php +++ b/module/VuFind/src/VuFind/Db/AdapterFactory.php @@ -136,6 +136,8 @@ public function getDriverName($type) switch (strtolower($type)) { case 'mysql': return 'mysqli'; + case 'mariadb': + return 'mysqli'; case 'oci8': return 'Oracle'; case 'pgsql':