[WIP] Fix database sync

.github/workflows/tests.yml

@@ -198,18 +198,9 @@ jobs:
   databasetests:
     name: Database tests
     runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:13-alpine
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
-        ports:
-          - 5432:5432
-        # needed because the postgres container does not provide a healthcheck
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
+      - name: Shutdown Ubuntu MySQL
+        run: sudo service mysql stop # Shutdown the Default MySQL to save memory, "sudo" is necessary, please do not remove it
       - name: Checkout source
         uses: actions/checkout@v4
         with:
@@ -228,24 +219,6 @@ jobs:
         run: ./gradlew databaseTest --rerun-tasks
         env:
           CI: "true"
-          DBMS: "postgresql"
-      - name: Shutdown Ubuntu MySQL
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Start custom MySQL
-        uses: mirromutth/[email protected]
-        with:
-          host port: 3800
-          container port: 3307
-          character set server: 'utf8'
-          collation server: 'utf8_general_ci'
-          mysql version: '8.0'
-          mysql database: 'jabref'
-          mysql root password: 'root'
-      - name: Run tests on MySQL
-        run: ./gradlew databaseTest --rerun-tasks
-        env:
-          CI: "true"
-          DBMS: "mysql"
 
   guitests:
     name: GUI tests

CHANGELOG.md

@@ -55,6 +55,7 @@
 - We improved the performance when pasting and importing entries in an existing library. [#11843](https://github.com/JabRef/jabref/pull/11843)
 - When fulltext search is selected but indexing is deactivated, a dialog is now shown asking if the user wants to enable indexing now [#9491](https://github.com/JabRef/jabref/issues/9491)
 - We changed instances of 'Search Selected' to 'Search Pre-configured' in Web Search Preferences UI. [#11871](https://github.com/JabRef/jabref/pull/11871)
+- We rewrote the [remote SQL database](https://docs.jabref.org/collaborative-work/sqldatabase) support. ⚠️database tables will be migrated. [#11879](https://github.com/JabRef/jabref/pull/11879) 
 - We added a new CSS style class `main-table` for the main table. [#11881](https://github.com/JabRef/jabref/pull/11881)
 
 ### Fixed
@@ -93,7 +94,7 @@
 - We removed support for case-sensitive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542)
 - We removed the description of search strings. [#11542](https://github.com/JabRef/jabref/pull/11542)
 - We removed support for importing using the SilverPlatterImporter (`Record INSPEC`). [#11576](https://github.com/JabRef/jabref/pull/11576)
-
+- We removed support for MySQL/MariaDB and Oracle.
 
 
 

build.gradle

@@ -205,18 +205,10 @@ dependencies {
 
     implementation 'com.fasterxml:aalto-xml:1.3.3'
 
-    implementation group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '2.7.9'
-
     implementation 'org.postgresql:postgresql:42.7.4'
 
     // Support unix socket connection types
     implementation 'com.kohlschutter.junixsocket:junixsocket-core:2.10.1'
-    implementation 'com.kohlschutter.junixsocket:junixsocket-mysql:2.10.0'
-
-    implementation ('com.oracle.ojdbc:ojdbc10:19.3.0.0') {
-        // causing module issues
-        exclude module: 'oraclepki'
-    }
 
     implementation ('com.google.guava:guava:33.1.0-jre') {
         // TODO: Remove this as soon as https://github.com/google/guava/issues/2960 is fixed
@@ -370,6 +362,9 @@ dependencies {
     // Even if "compileOnly" is used, IntelliJ always adds to module-info.java. To avoid issues during committing, we use "implementation" instead of "compileOnly"
     implementation 'io.github.adr:e-adr:2.0.0-SNAPSHOT'
 
+    implementation 'io.zonky.test:embedded-postgres:2.0.7'
+    implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:17.0.0')
+
     testImplementation 'io.github.classgraph:classgraph:4.8.176'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
     testImplementation 'org.junit.platform:junit-platform-launcher:1.10.3'
@@ -803,24 +798,8 @@ jlink {
         uses 'kong.unirest.core.json.JsonEngine'
         uses 'org.eclipse.jgit.lib.Signer'
         uses 'org.eclipse.jgit.transport.SshSessionFactory'
-        uses 'org.mariadb.jdbc.LocalInfileInterceptor'
-        uses 'org.mariadb.jdbc.authentication.AuthenticationPlugin'
-        uses 'org.mariadb.jdbc.credential.CredentialPlugin'
-        uses 'org.mariadb.jdbc.tls.TlsSocketPlugin'
 
-        provides 'org.mariadb.jdbc.tls.TlsSocketPlugin' with 'org.mariadb.jdbc.internal.protocol.tls.DefaultTlsSocketPlugin'
         provides 'java.sql.Driver' with 'org.postgresql.Driver'
-        provides 'org.mariadb.jdbc.authentication.AuthenticationPlugin' with 'org.mariadb.jdbc.internal.com.send.authentication.CachingSha2PasswordPlugin',
-                'org.mariadb.jdbc.internal.com.send.authentication.ClearPasswordPlugin',
-                'org.mariadb.jdbc.internal.com.send.authentication.Ed25519PasswordPlugin',
-                'org.mariadb.jdbc.internal.com.send.authentication.NativePasswordPlugin',
-                'org.mariadb.jdbc.internal.com.send.authentication.OldPasswordPlugin',
-                'org.mariadb.jdbc.internal.com.send.authentication.SendGssApiAuthPacket',
-                'org.mariadb.jdbc.internal.com.send.authentication.SendPamAuthPacket',
-                'org.mariadb.jdbc.internal.com.send.authentication.Sha256PasswordPlugin'
-        provides 'org.mariadb.jdbc.credential.CredentialPlugin' with 'org.mariadb.jdbc.credential.aws.AwsIamCredentialPlugin',
-                'org.mariadb.jdbc.credential.env.EnvCredentialPlugin',
-                'org.mariadb.jdbc.credential.system.PropertiesCredentialPlugin'
         provides 'java.security.Provider' with 'org.bouncycastle.jce.provider.BouncyCastleProvider',
                 'org.bouncycastle.pqc.jcajce.provider.BouncyCastlePQCProvider'
         provides 'kong.unirest.core.json.JsonEngine' with 'kong.unirest.modules.gson.GsonEngine';

docs/code-howtos/remote-storage-sql.md

@@ -9,14 +9,25 @@ For user documentation, see <https://docs.jabref.org/collaborative-work/sqldatab
 
 ## Handling large shared databases
 
-Synchronization times may get long when working with a large database containing several thousand entries. Therefore, synchronization only happens if several conditions are fulfilled:
+Synchronization times may get long when working with a large database containing several thousand entries.
+Therefore, we use PostgreSQL's `LISTEN` and `NOTIFY` commands to inform the client about changes in the database on an entry level.
+
+Background reading: <https://www.baeldung.com/spring-postgresql-message-broker>.
+
+## Handling synchronization of "micro-edits"
+
+It causes too much load both on the server and at all subscribed clients to synchronize every single letter change.
+Therefore, synchronization only happens if several conditions are fulfilled:
 
 * Edit to another field.
 * Major changes have been made (pasting or deleting more than one character).
 
 Class `org.jabref.logic.util.CoarseChangeFilter.java` checks both conditions.
 
-Remaining changes that have not been synchronized yet are saved at closing the database rendering additional closing time. Saving is realized in `org.jabref.logic.shared.DBMSSynchronizer.java`. Following methods account for synchronization modes:
+Remaining changes that have not been synchronized yet are saved at closing the database rendering additional closing time.
+Saving is realized in `org.jabref.logic.shared.DBMSSynchronizer.java`.
+
+Following methods account for synchronization modes:
 
 * `pullChanges` synchronizes the database unconditionally.
 * `pullLastEntryChanges` synchronizes only if there are remaining entry changes. It is invoked when closing the shared database (`closeSharedDatabase`).
@@ -61,3 +72,8 @@ PostgreSQL supports to register listeners on the database on changes.
 The listening is implemented at [`org.jabref.logic.shared.listener.PostgresSQLNotificationListener`](https://github.com/JabRef/jabref/blob/main/src/main/java/org/jabref/logic/shared/listener/PostgresSQLNotificationListener.java#L16).
 It "just" fetches updates from the server when a change occurred there.
 Thus, the changes are not actively pushed from the server, but still need to be fetched by the client.
+
+## Tests
+
+Tests are executed using [Zonky Embedded Postgres](https://github.com/zonkyio/embedded-postgres).
+This installs and runs a PostgreSQL server and frees the developer from the need to install a PostgreSQL server on the local machine.

src/main/java/module-info.java

@@ -97,10 +97,7 @@
     // endregion
 
     // region: SQL databases
-    requires ojdbc10;
     requires org.postgresql.jdbc;
-    requires org.mariadb.jdbc;
-    uses org.mariadb.jdbc.credential.CredentialPlugin;
     // endregion
 
     // region: Apache Commons and other (similar) helper libraries

src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -75,7 +75,6 @@ public enum StandardActions implements Action {
     REMOTE_DB(Localization.lang("Shared database"), IconTheme.JabRefIcons.REMOTE_DATABASE),
     EXPORT_SELECTED(Localization.lang("Export selected entries"), KeyBinding.EXPORT_SELECTED),
     CONNECT_TO_SHARED_DB(Localization.lang("Connect to shared database"), IconTheme.JabRefIcons.CONNECT_DB),
-    PULL_CHANGES_FROM_SHARED_DB(Localization.lang("Pull changes from shared database"), KeyBinding.PULL_CHANGES_FROM_SHARED_DATABASE),
     CLOSE_LIBRARY(Localization.lang("Close library"), Localization.lang("Close the current library"), IconTheme.JabRefIcons.CLOSE, KeyBinding.CLOSE_DATABASE),
     CLOSE_OTHER_LIBRARIES(Localization.lang("Close others"), Localization.lang("Close other libraries"), IconTheme.JabRefIcons.CLOSE),
     CLOSE_ALL_LIBRARIES(Localization.lang("Close all"), Localization.lang("Close all libraries"), IconTheme.JabRefIcons.CLOSE),

src/main/java/org/jabref/gui/frame/MainMenu.java

@@ -60,7 +60,6 @@
 import org.jabref.gui.push.PushToApplicationCommand;
 import org.jabref.gui.search.RebuildFulltextSearchIndexAction;
 import org.jabref.gui.shared.ConnectToSharedDatabaseCommand;
-import org.jabref.gui.shared.PullChangesFromSharedAction;
 import org.jabref.gui.sidepane.SidePane;
 import org.jabref.gui.sidepane.SidePaneType;
 import org.jabref.gui.slr.EditExistingStudyAction;
@@ -170,8 +169,7 @@ private void createMenu() {
                 new SeparatorMenuItem(),
 
                 factory.createSubMenu(StandardActions.REMOTE_DB,
-                        factory.createMenuItem(StandardActions.CONNECT_TO_SHARED_DB, new ConnectToSharedDatabaseCommand(frame, dialogService)),
-                        factory.createMenuItem(StandardActions.PULL_CHANGES_FROM_SHARED_DB, new PullChangesFromSharedAction(stateManager))),
+                        factory.createMenuItem(StandardActions.CONNECT_TO_SHARED_DB, new ConnectToSharedDatabaseCommand(frame, dialogService))),
 
                 new SeparatorMenuItem(),
 

src/main/java/org/jabref/gui/keyboard/KeyBinding.java

@@ -90,7 +90,6 @@ public enum KeyBinding {
     SHOW_PREFS("Preferences", Localization.lang("Preferences"), "ctrl+,", KeyBindingCategory.FILE),
     OPEN_URL_OR_DOI("Open URL or DOI", Localization.lang("Open URL or DOI"), "F3", KeyBindingCategory.TOOLS),
     PASTE("Paste", Localization.lang("Paste"), "ctrl+V", KeyBindingCategory.EDIT),
-    PULL_CHANGES_FROM_SHARED_DATABASE("Pull changes from shared database", Localization.lang("Pull changes from shared database"), "ctrl+shift+R", KeyBindingCategory.FILE),
     PREVIOUS_PREVIEW_LAYOUT("Previous preview layout", Localization.lang("Previous preview layout"), "shift+F9", KeyBindingCategory.VIEW),
     PREVIOUS_LIBRARY("Previous library", Localization.lang("Previous library"), "ctrl+PAGE_UP", KeyBindingCategory.VIEW),
     SCROLL_TO_NEXT_MATCH_CATEGORY("Scroll to next match category", Localization.lang("Scroll to next match category"), "right", KeyBindingCategory.VIEW),

src/main/java/org/jabref/gui/preferences/keybindings/presets/NewEntryBindingPreset.java

@@ -18,7 +18,6 @@ public Map<KeyBinding, String> getKeyBindings() {
         final Map<KeyBinding, String> keyBindings = new HashMap<>();
 
         // Clear conflicting default presets
-        keyBindings.put(KeyBinding.PULL_CHANGES_FROM_SHARED_DATABASE, "");
         keyBindings.put(KeyBinding.COPY_PREVIEW, "");
 
         // Add new entry presets

src/main/java/org/jabref/logic/exporter/MetaDataSerializer.java

@@ -31,7 +31,7 @@ private MetaDataSerializer() {
     }
 
     /**
-     * Writes all data in the format <key, serialized data>.
+     * Writes all data in the format {@code <key, serialized data>}.
      */
     public static Map<String, String> getSerializedStringMap(MetaData metaData,
                                                              GlobalCitationKeyPatterns globalCiteKeyPatterns) {

src/main/java/org/jabref/logic/shared/DBMSConnection.java

@@ -9,6 +9,7 @@
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.shared.exception.InvalidDBMSConnectionPropertiesException;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -19,6 +20,12 @@ public class DBMSConnection implements DatabaseConnection {
     private final Connection connection;
     private final DBMSConnectionProperties properties;
 
+    @VisibleForTesting
+    public DBMSConnection(Connection connection) {
+        this.connection = connection;
+        this.properties = null;
+    }
+
     public DBMSConnection(DBMSConnectionProperties connectionProperties) throws SQLException, InvalidDBMSConnectionPropertiesException {
         if (!connectionProperties.isValid()) {
             throw new InvalidDBMSConnectionPropertiesException();
@@ -38,7 +45,7 @@ public DBMSConnection(DBMSConnectionProperties connectionProperties) throws SQLE
             }
         } catch (SQLException e) {
             // Some systems like PostgreSQL retrieves 0 to every exception.
-            // Therefore a stable error determination is not possible.
+            // Therefore, a stable error determination is not possible.
             LOGGER.error("Could not connect to database: {} - Error code: {}", e.getMessage(), e.getErrorCode(), e);
             throw e;
         }