DD-1586 Added truncate-notifications with commandline input parsing

pom.xml

@@ -85,6 +85,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-csv</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

src/main/assembly/dist/cfg/example-config.yml

@@ -1,7 +1,13 @@
-dataverse:
+api:
   baseUrl: "http://localhost:8080"
   apiKey: "your-api-token"
 
+db:
+  host: localhost
+  database: "dvndb"
+  user: "dvnuser"
+  password: "dvnsecret"
+
 #
 # See https://www.dropwizard.io/en/latest/manual/configuration.html#logging
 #

src/main/java/nl/knaw/dans/dvcli/DdDataverseCli.java

@@ -17,6 +17,7 @@
 package nl.knaw.dans.dvcli;
 
 import lombok.extern.slf4j.Slf4j;
+import nl.knaw.dans.dvcli.action.Database;
 import nl.knaw.dans.dvcli.command.CollectionAssignRole;
 import nl.knaw.dans.dvcli.command.CollectionCmd;
 import nl.knaw.dans.dvcli.command.CollectionCreateDataset;
@@ -33,7 +34,9 @@
 import nl.knaw.dans.dvcli.command.CollectionView;
 import nl.knaw.dans.dvcli.command.DatasetCmd;
 import nl.knaw.dans.dvcli.command.DeleteDraft;
+import nl.knaw.dans.dvcli.command.NotificationTruncate;
 import nl.knaw.dans.dvcli.config.DdDataverseCliConfig;
+import nl.knaw.dans.lib.dataverse.DataverseClient;
 import nl.knaw.dans.lib.util.AbstractCommandLineApp;
 import nl.knaw.dans.lib.util.PicocliVersionProvider;
 import picocli.CommandLine;
@@ -56,7 +59,10 @@
     @Override
     public void configureCommandLine(CommandLine commandLine, DdDataverseCliConfig config) {
         log.debug("Building Dataverse client");
-        var dataverseClient = config.getDataverse().build();
+        var dataverseClient = config.getApi().build();
+        var databaseConfig = config.getDb();
+        var database = new Database(databaseConfig);
+
         commandLine.addSubcommand(new CommandLine(new CollectionCmd(dataverseClient))
                 .addSubcommand(new CollectionAssignRole())
                 .addSubcommand(new CollectionCreateDataset())
@@ -73,7 +79,8 @@
                 .addSubcommand(new CollectionView()))
             .addSubcommand(new CommandLine(new DatasetCmd(dataverseClient))
                 .addSubcommand(new DeleteDraft())
-            );
+            )
+            .addSubcommand(new CommandLine(new NotificationTruncate(database)));
         log.debug("Configuring command line");
     }
 }

src/main/java/nl/knaw/dans/dvcli/action/Database.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nl.knaw.dans.dvcli.action;
+
+import lombok.extern.slf4j.Slf4j;
+import nl.knaw.dans.dvcli.config.DdDataverseDatabaseConfig;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Provides access to the Dataverse Database (Postgres).
+ * Some actions are not supported by the Dataverse API (yet) 
+ * and must be done by direct access to the database.
+ * 
+ * Note that the sql input strings are not filtered in any way, 
+ * so don't put user input in there!
+ */
+@Slf4j
+public class Database {
+
+    public Database(DdDataverseDatabaseConfig config) {
+        this.host = config.getHost();
+        this.database = config.getDatabase();
+        this.user = config.getUser();
+        this.password = config.getPassword();
+    }
+
+    Connection connection = null;
+
+    String port = "5432"; // Fixed port for Postgres
+
+    String host;
+    String database;
+    String user;
+    String password;
+
+    public void connect() throws ClassNotFoundException, SQLException {
+            Class.forName("org.postgresql.Driver");
+            if (connection == null) {
+                log.debug("Starting connecting to database");
+                connection = DriverManager
+                        .getConnection("jdbc:postgresql://" + host + ":" + port + "/" + database,
+                                user,
+                                password);
+            }
+
+    }
+
+    public void close() {
+        try {
+            if (connection != null) {
+                log.debug("Close connection to database");
+                connection.close();
+            }
+        } catch (SQLException e) {
+            System.err.println( "Database error: " + e.getClass().getName() + " " + e.getMessage() );
+        } finally {
+            connection = null;
+        }
+    }
+
+    public List<List<String>> query(String sql) throws SQLException {
+        return query(sql, false);
+    }
+
+    public List<List<String>> query(String sql, Boolean startResultWithColumnNames) throws SQLException {
+        log.debug("Qerying database with: " + sql);
+
+        Statement stmt = connection.createStatement();
+        ResultSet rs = stmt.executeQuery( sql );
+        List<List<String>> rows = extractResult(rs, startResultWithColumnNames);
+
+        rs.close();
+        stmt.close();
+
+        return rows;
+    }
+
+    List<List<String>> extractResult(ResultSet rs, Boolean startResultWithColumnNames) throws SQLException {
+        List<List<String>> rows = new ArrayList<>();
+        // get column names
+        int numColumns = rs.getMetaData().getColumnCount();
+        if (startResultWithColumnNames) {
+            List<String> columnNames = new ArrayList<String>();
+            for (int i = 1; i <= numColumns; i++) {
+                columnNames.add(rs.getMetaData().getColumnName(i));
+            }
+            // make it the first row, for simplicity, a bit like with a csv file
+            rows.add(columnNames);
+        }
+
+        // get the data rows
+        while (rs.next()) {
+            List<String> row = new ArrayList<String>();
+            for (int i = 1; i <= numColumns; i++) {
+                row.add(rs.getString(i));
+            }
+            rows.add(row);
+        }
+        return rows;
+    }
+
+    public int update(String sql) throws SQLException {
+        log.debug("Updating database with: " + sql);
+        int rowCount = 0;
+
+        Statement stmt = connection.createStatement();
+        rowCount = stmt.executeUpdate( sql );
+        stmt.close();
+
+        return rowCount;
+    }
+}

src/main/java/nl/knaw/dans/dvcli/command/AbstractCmd.java

@@ -35,5 +35,5 @@ public Integer call() throws Exception {
         }
     }
 
-    public abstract void doCall() throws IOException, DataverseException;
+    public abstract void doCall() throws Exception, DataverseException;
 }

src/main/java/nl/knaw/dans/dvcli/command/NotificationTruncate.java

@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nl.knaw.dans.dvcli.command;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import nl.knaw.dans.dvcli.action.BatchProcessor;
+import nl.knaw.dans.dvcli.action.ConsoleReport;
+import nl.knaw.dans.dvcli.action.Database;
+import nl.knaw.dans.dvcli.action.Pair;
+import nl.knaw.dans.dvcli.action.ThrowingFunction;
+import nl.knaw.dans.dvcli.config.DdDataverseDatabaseConfig;
+
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+@CommandLine.Command(name = "truncate-notifications",
+        mixinStandardHelpOptions = true,
+        description = "Remove user notifications but keep up to a specified amount.",
+        sortOptions = false)
+@Slf4j
+public class NotificationTruncate extends AbstractCmd {
+    protected Database db;
+    public NotificationTruncate(@NonNull Database database) {
+        this.db = database;
+    }
+
+
+    @CommandLine.ArgGroup(exclusive = true, multiplicity = "1")
+    UserOptions users;
+
+    static class UserOptions {
+        @CommandLine.Option(names = { "--user" }, required = true, 
+                description = "The user whose notifications to truncate.")
+        int user; // a number, preventing accidental SQL injection
+        // This id is visible for 'superusers' in the Dataverse Dashboard
+
+        @CommandLine.Option(names = { "--all-users" }, required = true, 
+                description = "Truncate notifications for all users.")
+        boolean allUsers;
+    }
+
+    @CommandLine.Parameters(index = "0", paramLabel = "number-of-records-to-keep", 
+            description = "The number of notification records to keep.")
+    private int numberOfRecordsToKeep;
+
+    private record NotificationTruncateParams(Database db, int userId, int numberOfRecordsToKeep) {
+    }
+
+    protected <NotificationTruncateParams> BatchProcessor.BatchProcessorBuilder<NotificationTruncate.NotificationTruncateParams, String> paramsBatchProcessorBuilder() throws IOException {
+        return BatchProcessor.<NotificationTruncate.NotificationTruncateParams, String> builder();
+
+    }
+
+    private static class NotificationTruncateAction implements ThrowingFunction<NotificationTruncate.NotificationTruncateParams, String, Exception> {
+
+        @Override
+        public String apply(NotificationTruncateParams notificationTruncateParams) throws Exception {
+            // Dry-run; show what will be deleted
+            //List<List<String>> results = notificationTruncateParams.db.query(String.format("SELECT * FROM usernotification WHERE user_id = '%d' AND id NOT IN (SELECT id FROM usernotification WHERE user_id = '%d' ORDER BY senddate DESC LIMIT %d);", 
+            //        notificationTruncateParams.userId, notificationTruncateParams.userId, 
+            //        notificationTruncateParams.numberOfRecordsToKeep), true
+            //);
+            //return "Notifications for user " + notificationTruncateParams.userId + " that will be deleted: \n" 
+            //        + getResultsAsString(results);
+
+            // Actually delete the notifications
+            try {
+                log.info("Deleting notifications for user with id " + notificationTruncateParams.userId);
+                int rowCount = notificationTruncateParams.db.update(String.format("DELETE FROM usernotification WHERE user_id = '%d' AND id NOT IN (SELECT id FROM usernotification WHERE user_id = '%d' ORDER BY senddate DESC LIMIT %d);",
+                        notificationTruncateParams.userId, notificationTruncateParams.userId,
+                        notificationTruncateParams.numberOfRecordsToKeep));
+                return "Deleted " + rowCount + " record(s) for user with id " + notificationTruncateParams.userId;
+            } catch (SQLException e) {
+                throw new Exception("Error deleting notifications for user with id " + notificationTruncateParams.userId, e);
+            }
+        }
+    }
+
+    @Override
+    public void doCall() throws Exception {
+        // validate input
+        if (numberOfRecordsToKeep < 0) {
+            throw new Exception("Number of records to keep must be a positive integer, now it was " + numberOfRecordsToKeep + ".");
+        }
+
+        // Connect to database
+        //db = new Database(dbCfg);
+        db.connect();
+
+        paramsBatchProcessorBuilder()
+                .labeledItems(getItems())
+                .action(new NotificationTruncate.NotificationTruncateAction())
+                .delay(10L) // 10 ms, database should be able to handle this
+                .report(new ConsoleReport<>())
+                .build()
+                .process();
+
+        db.close(); // Not called if anny exception is thrown above!
+    }
+
+    protected List<Pair<String, NotificationTruncateParams>> getItems() throws Exception {
+        List<Pair<String, NotificationTruncateParams>> items = new ArrayList<>();
+        try {
+            if (users.allUsers) {
+                getUserIds(db).forEach(user_id -> items.add(new Pair<>(Integer.toString(user_id),
+                        new NotificationTruncateParams(db, user_id, numberOfRecordsToKeep))));
+            } else {
+                // single user
+                items.add(new Pair<>(Integer.toString(users.user),
+                        new NotificationTruncateParams(db, users.user, numberOfRecordsToKeep)));
+            }
+        } catch (SQLException e) {
+            throw new Exception("Error getting user ids: ", e);
+        }
+        return items;
+    }
+
+    // get the user_id for all users that need truncation
+    private List<Integer> getUserIds(Database db) throws SQLException {
+        List<Integer> users = new ArrayList<Integer>();
+        // Could just get all users with notifications
+        // String sql = "SELECT DISTINCT user_id FROM usernotification;";
+        // Instead we want only users with to many notifications
+        String sql = String.format("SELECT user_id FROM usernotification GROUP BY user_id HAVING COUNT(user_id) > %d;", numberOfRecordsToKeep);
+        List<List<String>> results = db.query(sql);
+        for (List<String> row : results) {
+            users.add(Integer.parseInt(row.get(0)));
+        }
+        return users;
+    }
+
+    private static String getResultsAsString(List<List<String>> results) {
+        // Note that the first row could be the column names
+        StringBuilder sb = new StringBuilder();
+        for (List<String> row : results) {
+            for (String cell : row) {
+                sb.append(cell).append(" ");
+            }
+            sb.append("\n");
+        }
+        return sb.toString();
+    }
+}

src/main/java/nl/knaw/dans/dvcli/config/DdDataverseCliConfig.java

@@ -19,10 +19,14 @@
 import io.dropwizard.core.Configuration;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.NonNull;
 import nl.knaw.dans.lib.util.DataverseClientFactory;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
 public class DdDataverseCliConfig extends Configuration {
-    private DataverseClientFactory dataverse;
+    private DataverseClientFactory api;
+
+    @NonNull
+    private DdDataverseDatabaseConfig db = new DdDataverseDatabaseConfig();
 }