-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
DD-1586 Added truncate-notifications with commandline input parsing (#6)
* Added truncate-notifications with commandline input parsing * Added database configuration for truncate-notifications * Initial database connection for truncate-notifications * Database connection for truncate-notifications working with a query * Database query refactoring for truncate-notifications * Implemented deleting notifications for specific user * Implemented deleting notifications for all users * Refactor notifications truncation, using batch processing * Throwing exceptions instead of just printing messages in notifications truncation * Initial implementation of unit tests for notification truncation * Refactoring the Database class; extracting result now in separate function * Added test for notification truncation of all users * Added db settings to example config * Rename the dataverse setting to api * Refactoring * Removing erroneous ';' in yml configs * Added @positive annotation to numberOfRecordsToKeep * Renamed test functions to snake-case * Remove redundant use of stdin manipulation in the NotificationTruncateTest * make sure database connection is closed, even if an exception is thrown * Using try-with-resources statements in Database query and update * Removing @positive annotation on numberOfRecordsToKeep * Clarify the user input parameter with the notification truncation * Added the delay option for notification truncate * Getting rid of most warnings * Have AbstractCmd doCall signature throw a general Exception
- Loading branch information
Showing
10 changed files
with
492 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| /* | ||
| * Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected]) | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package nl.knaw.dans.dvcli.action; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
| import nl.knaw.dans.dvcli.config.DdDataverseDatabaseConfig; | ||
|
|
||
| import java.sql.Connection; | ||
| import java.sql.DriverManager; | ||
| import java.sql.ResultSet; | ||
| import java.sql.SQLException; | ||
| import java.sql.Statement; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * Provides access to the Dataverse Database (Postgres). | ||
| * Some actions are not supported by the Dataverse API (yet) | ||
| * and must be done by direct access to the database. | ||
| * <p> | ||
| * Note that the sql input strings are not filtered in any way, | ||
| * so don't put user input in there! | ||
| */ | ||
| @Slf4j | ||
| public class Database { | ||
|
|
||
| public Database(DdDataverseDatabaseConfig config) { | ||
| this.host = config.getHost(); | ||
| this.database = config.getDatabase(); | ||
| this.user = config.getUser(); | ||
| this.password = config.getPassword(); | ||
| } | ||
|
|
||
| Connection connection = null; | ||
|
|
||
| String port = "5432"; // Fixed port for Postgres | ||
|
|
||
| String host; | ||
| String database; | ||
| String user; | ||
| String password; | ||
|
|
||
| public void connect() throws ClassNotFoundException, SQLException { | ||
| Class.forName("org.postgresql.Driver"); | ||
| if (connection == null) { | ||
| log.debug("Starting connecting to database"); | ||
| connection = DriverManager | ||
| .getConnection("jdbc:postgresql://" + host + ":" + port + "/" + database, | ||
| user, | ||
| password); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| public void close() { | ||
| try { | ||
| if (connection != null) { | ||
| log.debug("Close connection to database"); | ||
| connection.close(); | ||
| } | ||
| } catch (SQLException e) { | ||
| System.err.println( "Database error: " + e.getClass().getName() + " " + e.getMessage() ); | ||
| } finally { | ||
| connection = null; | ||
| } | ||
| } | ||
|
|
||
| public List<List<String>> query(String sql) throws SQLException { | ||
| return query(sql, false); | ||
| } | ||
|
|
||
| public List<List<String>> query(String sql, Boolean startResultWithColumnNames) throws SQLException { | ||
| log.debug("Querying database with: {}", sql); | ||
|
|
||
| try ( | ||
| Statement stmt = connection.createStatement(); | ||
| ResultSet rs = stmt.executeQuery(sql) | ||
| ) { | ||
| return extractResult(rs, startResultWithColumnNames); | ||
| } | ||
| } | ||
|
|
||
| List<List<String>> extractResult(ResultSet rs, Boolean startResultWithColumnNames) throws SQLException { | ||
| List<List<String>> rows = new ArrayList<>(); | ||
| // get column names | ||
| int numColumns = rs.getMetaData().getColumnCount(); | ||
| if (startResultWithColumnNames) { | ||
| List<String> columnNames = new ArrayList<String>(); | ||
| for (int i = 1; i <= numColumns; i++) { | ||
| columnNames.add(rs.getMetaData().getColumnName(i)); | ||
| } | ||
| // make it the first row, for simplicity, a bit like with a csv file | ||
| rows.add(columnNames); | ||
| } | ||
|
|
||
| // get the data rows | ||
| while (rs.next()) { | ||
| List<String> row = new ArrayList<String>(); | ||
| for (int i = 1; i <= numColumns; i++) { | ||
| row.add(rs.getString(i)); | ||
| } | ||
| rows.add(row); | ||
| } | ||
| return rows; | ||
| } | ||
|
|
||
| public int update(String sql) throws SQLException { | ||
| log.debug("Updating database with: {}", sql); | ||
|
|
||
| try (Statement stmt = connection.createStatement()) { | ||
| return stmt.executeUpdate(sql); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
162 changes: 162 additions & 0 deletions
162
src/main/java/nl/knaw/dans/dvcli/command/NotificationTruncate.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| /* | ||
| * Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected]) | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package nl.knaw.dans.dvcli.command; | ||
|
|
||
| import lombok.NonNull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import nl.knaw.dans.dvcli.action.BatchProcessor; | ||
| import nl.knaw.dans.dvcli.action.ConsoleReport; | ||
| import nl.knaw.dans.dvcli.action.Database; | ||
| import nl.knaw.dans.dvcli.action.Pair; | ||
| import nl.knaw.dans.dvcli.action.ThrowingFunction; | ||
|
|
||
| import picocli.CommandLine; | ||
|
|
||
| import java.io.IOException; | ||
| import java.sql.SQLException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| @CommandLine.Command(name = "truncate-notifications", | ||
| mixinStandardHelpOptions = true, | ||
| description = "Remove user notifications but keep up to a specified amount.", | ||
| sortOptions = false) | ||
| @Slf4j | ||
| public class NotificationTruncate extends AbstractCmd { | ||
| protected Database db; | ||
| public NotificationTruncate(@NonNull Database database) { | ||
| this.db = database; | ||
| } | ||
|
|
||
| private static final long DEFAULT_DELAY = 10L; // 10 ms, database should be able to handle this | ||
|
|
||
| @CommandLine.Option(names = { "-d", "--delay" }, description = "Delay in milliseconds between requests to the server (default: ${DEFAULT-VALUE}).", defaultValue = "" + DEFAULT_DELAY) | ||
| protected long delay = DEFAULT_DELAY; | ||
|
|
||
| @CommandLine.ArgGroup(exclusive = true, multiplicity = "1") | ||
| UserOptions users; | ||
|
|
||
| static class UserOptions { | ||
| @CommandLine.Option(names = { "--user" }, required = true, | ||
| description = "The user database id (a number) whose notifications to truncate.") | ||
| int user; // a number, preventing accidental SQL injection | ||
| // This id is visible for 'superusers' in the Dataverse Dashboard | ||
|
|
||
| @CommandLine.Option(names = { "--all-users" }, required = true, | ||
| description = "Truncate notifications for all users.") | ||
| boolean allUsers; | ||
| } | ||
|
|
||
| @CommandLine.Parameters(index = "0", paramLabel = "number-of-records-to-keep", | ||
| description = "The number of notification records to keep.") | ||
| private int numberOfRecordsToKeep; | ||
|
|
||
| private record NotificationTruncateParams(Database db, int userId, int numberOfRecordsToKeep) { | ||
| } | ||
|
|
||
| private BatchProcessor.BatchProcessorBuilder<NotificationTruncate.NotificationTruncateParams, String> paramsBatchProcessorBuilder() throws IOException { | ||
| return BatchProcessor.<NotificationTruncate.NotificationTruncateParams, String> builder(); | ||
| } | ||
|
|
||
| private static class NotificationTruncateAction implements ThrowingFunction<NotificationTruncate.NotificationTruncateParams, String, Exception> { | ||
|
|
||
| @Override | ||
| public String apply(NotificationTruncateParams notificationTruncateParams) throws Exception { | ||
| // Dry-run; show what will be deleted | ||
| //List<List<String>> results = notificationTruncateParams.db.query(String.format("SELECT * FROM usernotification WHERE user_id = '%d' AND id NOT IN (SELECT id FROM usernotification WHERE user_id = '%d' ORDER BY senddate DESC LIMIT %d);", | ||
| // notificationTruncateParams.userId, notificationTruncateParams.userId, | ||
| // notificationTruncateParams.numberOfRecordsToKeep), true | ||
| //); | ||
| //return "Notifications for user " + notificationTruncateParams.userId + " that will be deleted: \n" | ||
| // + getResultsAsString(results); | ||
|
|
||
| // Actually delete the notifications | ||
| try { | ||
| log.info("Deleting notifications for user with id {}", notificationTruncateParams.userId); | ||
| int rowCount = notificationTruncateParams.db.update(String.format("DELETE FROM usernotification WHERE user_id = '%d' AND id NOT IN (SELECT id FROM usernotification WHERE user_id = '%d' ORDER BY senddate DESC LIMIT %d);", | ||
| notificationTruncateParams.userId, notificationTruncateParams.userId, | ||
| notificationTruncateParams.numberOfRecordsToKeep)); | ||
| return "Deleted " + rowCount + " record(s) for user with id " + notificationTruncateParams.userId; | ||
| } catch (SQLException e) { | ||
| throw new Exception("Error deleting notifications for user with id " + notificationTruncateParams.userId, e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void doCall() throws Exception { | ||
| // validate input | ||
| if (numberOfRecordsToKeep < 0) { | ||
| throw new Exception("Number of records to keep must be a positive integer, now it was " + numberOfRecordsToKeep + "."); | ||
| } | ||
|
|
||
| db.connect(); | ||
| try { | ||
| paramsBatchProcessorBuilder() | ||
| .labeledItems(getItems()) | ||
| .action(new NotificationTruncate.NotificationTruncateAction()) | ||
| .delay(delay) | ||
| .report(new ConsoleReport<>()) | ||
| .build() | ||
| .process(); | ||
| } finally { | ||
| db.close(); | ||
| } | ||
| } | ||
|
|
||
| List<Pair<String, NotificationTruncateParams>> getItems() throws Exception { | ||
| List<Pair<String, NotificationTruncateParams>> items = new ArrayList<>(); | ||
| try { | ||
| if (users.allUsers) { | ||
| getUserIds(db).forEach(user_id -> items.add(new Pair<>(Integer.toString(user_id), | ||
| new NotificationTruncateParams(db, user_id, numberOfRecordsToKeep)))); | ||
| } else { | ||
| // single user | ||
| items.add(new Pair<>(Integer.toString(users.user), | ||
| new NotificationTruncateParams(db, users.user, numberOfRecordsToKeep))); | ||
| } | ||
| } catch (SQLException e) { | ||
| throw new Exception("Error getting user ids: ", e); | ||
| } | ||
| return items; | ||
| } | ||
|
|
||
| // get the user_id for all users that need truncation | ||
| private List<Integer> getUserIds(Database db) throws SQLException { | ||
| List<Integer> users = new ArrayList<Integer>(); | ||
| // Could just get all users with notifications | ||
| // String sql = "SELECT DISTINCT user_id FROM usernotification;"; | ||
| // Instead we want only users with to many notifications | ||
| String sql = String.format("SELECT user_id FROM usernotification GROUP BY user_id HAVING COUNT(user_id) > %d;", numberOfRecordsToKeep); | ||
| List<List<String>> results = db.query(sql); | ||
| for (List<String> row : results) { | ||
| users.add(Integer.parseInt(row.get(0))); | ||
| } | ||
| return users; | ||
| } | ||
|
|
||
| private static String getResultsAsString(List<List<String>> results) { | ||
| // Note that the first row could be the column names | ||
| StringBuilder sb = new StringBuilder(); | ||
| for (List<String> row : results) { | ||
| for (String cell : row) { | ||
| sb.append(cell).append(" "); | ||
| } | ||
| sb.append("\n"); | ||
| } | ||
| return sb.toString(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.