Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixed first/last_indexed, improved import perf, support for mariadb connector #4108

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Fixed first/last_indexed, improved import perf, support for mariadb connector #4108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/vufind/config.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -817,7 +817,7 @@ url_shortener_key_type = md5
[Database]
; Connection string format is [platform]://[username]:[password]@[host]:[port]/[db]
; where:
; [platform/driver] = database platform (mysql, oci8 or pgsql)
; [platform/driver] = database platform (mysql, mariadb, oci8 or pgsql)
; [username] = username for connection
; [password] = password for connection (optional)
; [host] = host of database server
Expand Down
40 changes: 38 additions & 2 deletions import/index_java/src/org/vufind/index/DatabaseManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ public class DatabaseManager
// Shutdown flag:
private boolean shuttingDown = false;

private UpdateDateTracker udt;

private static ThreadLocal<DatabaseManager> managerCache =
new ThreadLocal<DatabaseManager>()
{
Expand Down Expand Up @@ -102,9 +104,11 @@ private void connectToDatabaseUsingStringConfig() throws Throwable

// Parse key settings from the PHP-style DSN:
String platform = "invalid";
if (dsn.substring(0, 8).equals("mysql://")) {
if (dsn.startsWith("mysql://")) {
platform = "mysql";
} else if (dsn.substring(0, 8).equals("pgsql://")) {
} else if (dsn.startsWith("mariadb://")) {
platform = "mariadb";
} else if (dsn.startsWith("pgsql://")) {
platform = "postgresql";
}

Expand Down Expand Up @@ -172,13 +176,37 @@ private void connectToDatabaseUsingParams(String platform, String host, String p
prefix = "mysql";
String useSsl = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "use_ssl", false) ? "true" : "false";
extraParams = "?useSSL=" + useSsl;
// NOTE: useSSL is deprecated for both mysql and mariadb
// sslMode is supported since mysql Connector/J 8.0.13 and mariadb Connector/J 3.0.0
if (useSsl != "false") {
String verifyCert = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "verify_server_certificate", false) ? "true" : "false";
extraParams += "&verifyServerCertificate=" + verifyCert;
}
extraParams += "&rewriteBatchedStatements=true";
} else if (platform.equals("mariadb")) {
classname = "org.mariadb.jdbc.Driver";
prefix = "mariadb";
boolean useSsl = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "use_ssl", false);
String sslMode;
if (useSsl) {
boolean verifyCert = ConfigManager.instance().getBooleanConfigSetting("config.ini", "Database", "verify_server_certificate", false);
if (verifyCert) {
sslMode = "verify-full";
// NOTE: we could also use "verify-ca" here, it would be better to have an sslMode VuFind config
// (but note that mysql values are different)
} else {
sslMode = "trust";
}
} else {
sslMode = "disable";
}
extraParams = "?sslMode=" + sslMode;
// NOTE: rewriteBatchedStatements was removed from mariadb Connector/J in version 3.0.0.
// It was replaced by useBulkStmts, which defaults to true.
} else if (platform.equals("pgsql") || platform.equals("postgresql")) {
classname = "org.postgresql.Driver";
prefix = "postgresql";
extraParams = "?reWriteBatchedInserts=true";
}

Class.forName(classname).getDeclaredConstructor().newInstance();
Expand All @@ -194,6 +222,10 @@ private void connectToDatabaseUsingParams(String platform, String host, String p
Runtime.getRuntime().addShutdownHook(new DatabaseManagerShutdownThread(this));
}

void setUpdateDateTracker(UpdateDateTracker udt) {
this.udt = udt;
}

private void disconnectFromDatabase()
{
if (vufindDatabase != null) {
Expand All @@ -208,6 +240,10 @@ private void disconnectFromDatabase()

public void shutdown()
{
if (udt != null) {
// We can't use UpdateDateTracker.instance() in shutdown, so we have to use a previously-saved reference
udt.shutdown();
}
disconnectFromDatabase();
shuttingDown = true;
}
Expand Down
135 changes: 86 additions & 49 deletions import/index_java/src/org/vufind/index/UpdateDateTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,19 @@
import java.time.format.DateTimeFormatter;
import java.time.LocalDateTime;

import org.apache.log4j.Logger;

/**
* Class for managing record update dates.
*/
public class UpdateDateTracker
{
private static Logger logger = Logger.getLogger(UpdateDateTracker.class);

private static final int BATCH_SIZE = 100;
private int insertBatchCount = 0;
private int updateBatchCount = 0;

private Connection db;
private String core;
private String id;
Expand All @@ -37,14 +45,21 @@ public class UpdateDateTracker
private Timestamp lastRecordChange;
private Timestamp deleted;

PreparedStatement insertSql;
PreparedStatement selectSql;
PreparedStatement updateSql;

private static ThreadLocal<UpdateDateTracker> trackerCache =
new ThreadLocal<UpdateDateTracker>()
{
@Override
protected UpdateDateTracker initialValue()
{
try {
return new UpdateDateTracker(DatabaseManager.instance().getConnection());
DatabaseManager dbm = DatabaseManager.instance();
UpdateDateTracker udt = new UpdateDateTracker(dbm.getConnection());
dbm.setUpdateDateTracker(udt);
return udt;
} catch (SQLException e) {
throw new RuntimeException(e.getMessage());
}
Expand All @@ -56,6 +71,25 @@ public static UpdateDateTracker instance()
return trackerCache.get();
}

private void possiblyExecuteBatch(boolean update, PreparedStatement statement, boolean force) throws SQLException
{
int count = update ? updateBatchCount : insertBatchCount;
if (count >= BATCH_SIZE || (count > 0 && force)) {
try {
statement.executeBatch();
db.commit();
} catch (SQLException ex) {
logger.error("SQLException in possiblyExecuteBatch(): " + ex.getMessage());
throw ex;
}
if (update) {
updateBatchCount = 0;
} else {
insertBatchCount = 0;
}
}
}

/* Private support method: create a row in the change_tracker table.
*/
private void createRow(Timestamp newRecordChange) throws SQLException
Expand All @@ -65,46 +99,32 @@ private void createRow(Timestamp newRecordChange) throws SQLException
lastRecordChange = newRecordChange;

// Save new values to the database:
try (
PreparedStatement insertSql = db.prepareStatement(
"INSERT INTO change_tracker(core, id, first_indexed, last_indexed, last_record_change) " +
"VALUES(?, ?, ?, ?, ?);"
);
) {
insertSql.setString(1, core);
insertSql.setString(2, id);
insertSql.setTimestamp(3, firstIndexed);
insertSql.setTimestamp(4, lastIndexed);
insertSql.setTimestamp(5, lastRecordChange);
insertSql.executeUpdate();
}
insertSql.setString(1, core);
insertSql.setString(2, id);
insertSql.setTimestamp(3, firstIndexed);
insertSql.setTimestamp(4, lastIndexed);
insertSql.setTimestamp(5, lastRecordChange);
insertSql.addBatch();
insertBatchCount++;
possiblyExecuteBatch(false, insertSql, false);
}

/* Private support method: read a row from the change_tracker table.
*/
private boolean readRow() throws SQLException
{
try (
PreparedStatement selectSql = db.prepareStatement(
"SELECT first_indexed, last_indexed, last_record_change, deleted " +
"FROM change_tracker WHERE core = ? AND id = ?;",
ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY
)
) {
selectSql.setString(1, core);
selectSql.setString(2, id);
try (ResultSet result = selectSql.executeQuery()) {
// No results? Return false:
if (!result.first()) {
return false;
} else {
// If we got this far, we have results -- load them into the object:
firstIndexed = result.getTimestamp(1);
lastIndexed = result.getTimestamp(2);
lastRecordChange = result.getTimestamp(3);
deleted = result.getTimestamp(4);
}
selectSql.setString(1, core);
selectSql.setString(2, id);
try (ResultSet result = selectSql.executeQuery()) {
// No results? Free resources and return false:
if (!result.first()) {
return false;
}
// If we got this far, we have results -- load them into the object:
firstIndexed = result.getTimestamp(1);
lastIndexed = result.getTimestamp(2);
lastRecordChange = result.getTimestamp(3);
deleted = result.getTimestamp(4);
}
return true;
}
Expand All @@ -123,28 +143,45 @@ private void updateRow(Timestamp newRecordChange) throws SQLException
lastRecordChange = newRecordChange;

// Save new values to the database:
try (
PreparedStatement updateSql = db.prepareStatement(
"UPDATE change_tracker " +
"SET first_indexed = ?, last_indexed = ?, last_record_change = ?, deleted = ? " +
"WHERE core = ? AND id = ?;"
)
) {
updateSql.setTimestamp(1, firstIndexed);
updateSql.setTimestamp(2, lastIndexed);
updateSql.setTimestamp(3, lastRecordChange);
updateSql.setNull(4, java.sql.Types.NULL);
updateSql.setString(5, core);
updateSql.setString(6, id);
updateSql.executeUpdate();
}
updateSql.setTimestamp(1, firstIndexed);
updateSql.setTimestamp(2, lastIndexed);
updateSql.setTimestamp(3, lastRecordChange);
updateSql.setNull(4, java.sql.Types.NULL);
updateSql.setString(5, core);
updateSql.setString(6, id);
updateSql.addBatch();
updateBatchCount++;
possiblyExecuteBatch(true, updateSql, false);
}

/* Constructor:
*/
public UpdateDateTracker(Connection dbConnection) throws SQLException
{
db = dbConnection;
db.setAutoCommit(false);
insertSql = db.prepareStatement(
"INSERT INTO change_tracker(core, id, first_indexed, last_indexed, last_record_change) " +
"VALUES(?, ?, ?, ?, ?);");
selectSql = db.prepareStatement(
"SELECT first_indexed, last_indexed, last_record_change, deleted " +
"FROM change_tracker WHERE core = ? AND id = ?;",
ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
updateSql = db.prepareStatement("UPDATE change_tracker " +
"SET first_indexed = ?, last_indexed = ?, last_record_change = ?, deleted = ? " +
"WHERE core = ? AND id = ?;");
}

void shutdown() {
try {
possiblyExecuteBatch(false, insertSql, true);
possiblyExecuteBatch(true, updateSql, true);
insertSql.close();
selectSql.close();
updateSql.close();
} catch (SQLException ex) {
logger.error("SQLException in shutdown hook: " + ex.getMessage());
}
}

/* Get the first indexed date (IMPORTANT: index() must be called before this method)
Expand Down
Binary file added import/lib_local/mariadb-java-client-3.5.1.jar
View file
Open in desktop
Binary file not shown.
2 changes: 2 additions & 0 deletions module/VuFind/src/VuFind/Db/AdapterFactory.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ public function getDriverName($type)
switch (strtolower($type)) {
case 'mysql':
return 'mysqli';
case 'mariadb':
return 'mysqli';
case 'oci8':
return 'Oracle';
case 'pgsql':
Expand Down