From 89b65f96abeffb6c3b1d8b609582e7e47c2a8665 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 20 Apr 2023 12:21:49 +0200 Subject: [PATCH 01/58] Add basic housekeeping module, and scaffolding for cxz generation --- housekeeping/build.gradle | 54 ++++++++++++ .../whelk/housekeeping/CXZGenerator.groovy | 83 +++++++++++++++++++ .../whelk/housekeeping/WebInterface.groovy | 70 ++++++++++++++++ housekeeping/src/main/webapp/WEB-INF/web.xml | 21 +++++ .../00000021-add-notice-table.plsql | 41 +++++++++ settings.gradle | 2 +- 6 files changed, 270 insertions(+), 1 deletion(-) create mode 100755 housekeeping/build.gradle create mode 100644 housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy create mode 100755 housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy create mode 100755 housekeeping/src/main/webapp/WEB-INF/web.xml create mode 100755 librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql diff --git a/housekeeping/build.gradle b/housekeeping/build.gradle new file mode 100755 index 0000000000..c4cfc09e6a --- /dev/null +++ b/housekeeping/build.gradle @@ -0,0 +1,54 @@ +apply plugin: 'war' +apply plugin: 'groovy' +apply from: '../gretty.plugin' + +repositories { + mavenCentral() +} + +// Don't blame me for this TRAVESTY. It is a necessity because of the versioning of xml-apis (2.0.2 which gradle otherwise chooses is OLDER (and broken) despite the version.) +configurations.all { + resolutionStrategy { + force "xml-apis:xml-apis:1.4.01" + } +} + +sourceSets { + main { + java { srcDirs = [] } + groovy { srcDirs = ['src/main/java', 'src/main/groovy'] } + } + test { + groovy { srcDir 'src/test/groovy/' } + } +} + + +dependencies { + // XL dependencies + implementation(project(':whelk-core')) + + // Logging + implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: "${log4jVersion}" + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: "${log4jVersion}" + + // metrics + implementation "io.prometheus:simpleclient:${prometheusVersion}" + implementation "io.prometheus:simpleclient_servlet:${prometheusVersion}" + + //implementation 'org.apache.commons:commons-lang3:3.3.2' + implementation "org.codehaus.groovy:groovy:${groovyVersion}" +} + +gretty { + jvmArgs = ['-XX:+UseParallelGC'] + systemProperties = ['xl.secret.properties': System.getProperty("xl.secret.properties")] + httpPort = 8589 + scanInterval = 0 + afterEvaluate { + appRunDebug { + debugPort = 5006 + debugSuspend = false + } + } +} diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy new file mode 100644 index 0000000000..e831b62e7d --- /dev/null +++ b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy @@ -0,0 +1,83 @@ +package whelk.housekeeping + +import whelk.Whelk +import groovy.transform.CompileStatic +import groovy.util.logging.Log4j2 as Log + +@CompileStatic +@Log +class CXZGenerator extends HouseKeeper { + + public CXZGenerator(Whelk whelk) { + + } + + public String getName() { + return "CXZ notifier generator" + } + + public String getStatusDescription() { + return "OK" + } + + public void trigger() { + System.err.println(" ** INTERNAL TICK **") + } + + + //zonedFrom = ZonedDateTime.parse(from); + //zonedUntil = ZonedDateTime.parse(until); + /*public void generate(ZonedDateTime zonedFrom, ZonedDateTime zonedUntil) throws SQLException { + + Connection connection; + PreparedStatement statement; + ResultSet resultSet; + + connection = whelk.getStorage().getOuterConnection(); + try { + String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth', 'hold') AND ( modified BETWEEN ? AND ? );"; + connection.setAutoCommit(false); + statement = connection.prepareStatement(sql); + statement.setTimestamp(1, new Timestamp(zonedFrom.toInstant().getEpochSecond() * 1000L)); + statement.setTimestamp(2, new Timestamp(zonedUntil.toInstant().getEpochSecond() * 1000L)); + statement.setFetchSize(512); + resultSet = statement.executeQuery(); + } catch (Throwable e) { + connection.close(); + throw e; + } + }*/ + + +} + + +/* + if (fromVersion < 0 || fromVersion >= untilVersion) + return // error out + + // We don't need all versions, just specifically the starting and end point of the + // sought interval. + List versions = whelk.getStorage().loadDocumentHistory(id) + List relevantVersions = [] + relevantVersions.add(versions.get(fromVersion)) + relevantVersions.add(versions.get(untilVersion)) + + History history = new History(relevantVersions, whelk.getJsonld()) + Map changes = history.m_changeSetsMap.changeSets[1] + System.err.println("added:\n\t" + changes.addedPaths) + System.err.println("removed:\n\t" + changes.removedPaths) + */ + + +/* +void doGet(HttpServletRequest request, HttpServletResponse response) { +{ + Map userInfo = request.getAttribute("user") + if (!isValidUserWithPermission(request, response, userInfo)) + return + + String id = "${userInfo.id}".digest(ID_HASH_FUNCTION) + String data = whelk.getUserData(id) ?: upgradeOldEmailBasedEntry(userInfo) ?: "{}" +} + */ diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy new file mode 100755 index 0000000000..68206bcfe5 --- /dev/null +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -0,0 +1,70 @@ +package whelk.housekeeping + +import whelk.Whelk; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import groovy.transform.CompileStatic +import groovy.util.logging.Log4j2 as Log + +import java.time.ZonedDateTime + +@CompileStatic +public abstract class HouseKeeper { + public abstract String getName() + public abstract String getStatusDescription() + public abstract void trigger() + + public ZonedDateTime lastFailAt = null +} + +@CompileStatic +@Log +public class WebInterface extends HttpServlet { + private static final long PERIODIC_TRIGGER_MS = 10 * 1000; + private final Timer timer = new Timer("Housekeeper-timer", true) + private List houseKeepers = [] + + public void init() { + Whelk whelk = Whelk.createLoadedCoreWhelk() + + houseKeepers = [] + houseKeepers.add(new CXZGenerator(whelk)) + + for (HouseKeeper hk : houseKeepers) { + timer.scheduleAtFixedRate({ + try { + hk.trigger() + } catch (Throwable e) { + log.error("Could not handle throwable in Housekeeper TimerTask.", e) + hk.lastFailAt = ZonedDateTime.now() + } + }, PERIODIC_TRIGGER_MS, PERIODIC_TRIGGER_MS) + } + + } + + public void destroy() { + } + + public void doGet(HttpServletRequest req, HttpServletResponse res) { + StringBuilder sb = new StringBuilder() + sb.append("Active housekeepers: " + houseKeepers.size() + "\n") + sb.append("--------------\n") + for (HouseKeeper hk : houseKeepers) { + sb.append(hk.getName() + "\n") + if (hk.lastFailAt) + sb.append("last failed at: " + hk.lastFailAt + "\n") + else + sb.append("no failures\n") + sb.append("status:\n") + sb.append(hk.statusDescription+"\n") + sb.append("--------------\n") + } + //res.sendStatus(HttpServletResponse.SC_ACCEPTED, sb.toString()) + //res.sendError(HttpServletResponse.SC_BAD_REQUEST, "CHYIIL.. DUDE"); + res.setStatus(HttpServletResponse.SC_OK) + res.setContentType("text/plain") + res.getOutputStream().print(sb.toString()) + } +} diff --git a/housekeeping/src/main/webapp/WEB-INF/web.xml b/housekeeping/src/main/webapp/WEB-INF/web.xml new file mode 100755 index 0000000000..669dcf68ce --- /dev/null +++ b/housekeeping/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,21 @@ + + + + Housekeeping + + + HousekeepingServlet + whelk.housekeeping.WebInterface + 1 + + + + HousekeepingServlet + / + + + diff --git a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql new file mode 100755 index 0000000000..ba2981fa43 --- /dev/null +++ b/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql @@ -0,0 +1,41 @@ +BEGIN; + +DO $$DECLARE + -- THESE MUST BE CHANGED WHEN YOU COPY THE SCRIPT! + + -- The version you expect the database to have _before_ the migration + old_version numeric := 20; + -- The version the database should have _after_ the migration + new_version numeric := 21; + + -- hands off + existing_version numeric; + +BEGIN + + -- Check existing version +SELECT version from lddb__schema INTO existing_version; +IF ( existing_version <> old_version) THEN + RAISE EXCEPTION 'ASKED TO MIGRATE FROM INCORRECT EXISTING VERSION!'; +ROLLBACK; +END IF; +UPDATE lddb__schema SET version = new_version; + +-- ACTUAL SCHEMA CHANGES HERE: + +ALTER TABLE lddb__versions ADD UNIQUE (pk); +CREATE TABLE IF NOT EXISTS lddb__notices ( + pk SERIAL PRIMARY KEY, + versionid INTEGER, + userid TEXT, + handled BOOLEAN DEFAULT FALSE, + created timestamp with time zone DEFAULT now() NOT NULL, + + CONSTRAINT version_fk FOREIGN KEY (versionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, + CONSTRAINT user_fk FOREIGN KEY (userid) REFERENCES lddb__user_data(id) ON DELETE CASCADE +); +CREATE INDEX idx_notices ON lddb__notices USING BTREE (userid); + +END$$; + +COMMIT; diff --git a/settings.gradle b/settings.gradle index eff22917f9..f7856044e8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ rootProject.name = 'librisxl' include('apix_export', 'apix_server', 'batchimport', 'importers', 'marc_export', 'oaipmh', 'rest', 'whelk-core', 'whelktool', - 'gui-whelktool', 'trld-java') + 'gui-whelktool', 'trld-java', 'housekeeping') From 0c7021e4a9ef299ed85b31ac911dbf479e4f64a5 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 24 Apr 2023 13:07:38 +0200 Subject: [PATCH 02/58] list changed records --- .../whelk/housekeeping/CXZGenerator.groovy | 77 ++++++++++++++++++- .../00000021-add-notice-table.plsql | 3 +- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy index e831b62e7d..0cbefa7d20 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy @@ -3,13 +3,27 @@ package whelk.housekeeping import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log +import whelk.history.DocumentVersion +import whelk.history.History + +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.Timestamp +import java.time.Instant +import java.time.ZonedDateTime +import java.time.temporal.ChronoUnit +import java.time.temporal.TemporalUnit @CompileStatic @Log class CXZGenerator extends HouseKeeper { - public CXZGenerator(Whelk whelk) { + private String status = "OK" + private Whelk whelk + public CXZGenerator(Whelk whelk) { + this.whelk = whelk } public String getName() { @@ -17,11 +31,68 @@ class CXZGenerator extends HouseKeeper { } public String getStatusDescription() { - return "OK" + return status } public void trigger() { - System.err.println(" ** INTERNAL TICK **") + + Connection connection + PreparedStatement statement + ResultSet resultSet + + connection = whelk.getStorage().getOuterConnection() + connection.setAutoCommit(false) + try { + + // First, determine the time interval of changes for which to generate notices. + // This interval, should generally be: From the last generated notice until now. + // However, if there are no previously generated notices (near enough in time), use + // now - [some pre set value], to avoid scanning the whole catalog. + String sql = "SELECT MAX(created) FROM lddb__notices;" + statement = connection.prepareStatement(sql) + resultSet = statement.executeQuery() + Timestamp from = Timestamp.from(Instant.now().minus(20, ChronoUnit.DAYS)) + if (resultSet.next()) { + Timestamp lastCreated = resultSet.getTimestamp(1) + if (lastCreated && lastCreated.after(from)) + from = lastCreated + } + Timestamp until = Timestamp.from(Instant.now()) + + // Then fetch all changes within that interval + sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth', 'hold') AND ( modified BETWEEN ? AND ? );"; + connection.setAutoCommit(false); + statement = connection.prepareStatement(sql) + statement.setTimestamp(1, from) + statement.setTimestamp(2, until) + statement.setFetchSize(512) + resultSet = statement.executeQuery() + while (resultSet.next()) { + String id = resultSet.getString("id") + + /* + List versions = whelk.getStorage().loadDocumentHistory(id) + List relevantVersions = [] + relevantVersions.add(versions.get(fromVersion)) + relevantVersions.add(versions.get(untilVersion)) + + */ + System.err.println("Was changed: " + id) + + //History history = new History(relevantVersions, whelk.getJsonld()) + + //Map changes = history.m_changeSetsMap.changeSets[1] + //System.err.println("added:\n\t" + changes.addedPaths) + //System.err.println("removed:\n\t" + changes.removedPaths) + } + + } catch (Throwable e) { + status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() + throw e + } finally { + connection.close() + } + } diff --git a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql index ba2981fa43..1de62d84c2 100755 --- a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql +++ b/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql @@ -34,7 +34,8 @@ CREATE TABLE IF NOT EXISTS lddb__notices ( CONSTRAINT version_fk FOREIGN KEY (versionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, CONSTRAINT user_fk FOREIGN KEY (userid) REFERENCES lddb__user_data(id) ON DELETE CASCADE ); -CREATE INDEX idx_notices ON lddb__notices USING BTREE (userid); +CREATE INDEX idx_notices_user ON lddb__notices USING BTREE (userid); +CREATE INDEX idx_notices_created ON lddb__notices USING BTREE (created); END$$; From fcba09246360cfa816bfb3587a3df85b4c115704 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 26 Apr 2023 09:27:47 +0200 Subject: [PATCH 03/58] Temporary progress towards CXZ --- .../groovy/whelk/importer/MergeSpec.groovy | 34 ++++++------ .../whelk/housekeeping/CXZGenerator.groovy | 34 +++++++++--- .../whelk/housekeeping/WebInterface.groovy | 8 ++- .../00000021-add-notice-table.plsql | 1 + .../groovy/whelk/rest/api/CrudSpec.groovy | 4 +- .../component/PostgreSQLComponent.groovy | 55 ++++++++++++++++++- .../groovy/whelk/history/DocumentVersion.java | 4 +- .../groovy/whelk/history/HistorySpec.groovy | 8 +-- 8 files changed, 112 insertions(+), 36 deletions(-) diff --git a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy index 1270ee4bb8..c57faeed5d 100644 --- a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy +++ b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy @@ -55,7 +55,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -101,7 +101,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -147,7 +147,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -185,7 +185,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) @@ -224,7 +224,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -262,7 +262,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -303,7 +303,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -348,7 +348,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) @@ -395,7 +395,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -441,7 +441,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -479,7 +479,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) @@ -533,7 +533,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -614,7 +614,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -695,7 +695,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -776,7 +776,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -882,7 +882,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -986,7 +986,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) def incoming = new Document( (Map) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy index 0cbefa7d20..ee04ebdf20 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy @@ -51,7 +51,7 @@ class CXZGenerator extends HouseKeeper { String sql = "SELECT MAX(created) FROM lddb__notices;" statement = connection.prepareStatement(sql) resultSet = statement.executeQuery() - Timestamp from = Timestamp.from(Instant.now().minus(20, ChronoUnit.DAYS)) + Timestamp from = Timestamp.from(Instant.now().minus(2, ChronoUnit.DAYS)) if (resultSet.next()) { Timestamp lastCreated = resultSet.getTimestamp(1) if (lastCreated && lastCreated.after(from)) @@ -61,7 +61,7 @@ class CXZGenerator extends HouseKeeper { // Then fetch all changes within that interval sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth', 'hold') AND ( modified BETWEEN ? AND ? );"; - connection.setAutoCommit(false); + connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) statement.setTimestamp(2, until) @@ -70,20 +70,36 @@ class CXZGenerator extends HouseKeeper { while (resultSet.next()) { String id = resultSet.getString("id") - /* + // "versions" come sorted by ascending modification time, so oldest version first. + // We want to pick the "from version" (the base for which this notice details changes) + // as the last saved version *before* the sought interval. + DocumentVersion fromVersion = null List versions = whelk.getStorage().loadDocumentHistory(id) + for (DocumentVersion version : versions) { + if (version.doc.getModifiedTimestamp().isBefore(from.toInstant())) + fromVersion = version + } + if (fromVersion == null) + continue + + DocumentVersion untilVersion = versions.last() + if (untilVersion == fromVersion) + continue + List relevantVersions = [] - relevantVersions.add(versions.get(fromVersion)) - relevantVersions.add(versions.get(untilVersion)) + relevantVersions.add(fromVersion) + relevantVersions.add(untilVersion) - */ - System.err.println("Was changed: " + id) + System.err.println("Was changed: " + id + " spans: " + fromVersion.doc.getModified() + " -> " + untilVersion.doc.getModified()) - //History history = new History(relevantVersions, whelk.getJsonld()) + History history = new History(relevantVersions, whelk.getJsonld()) - //Map changes = history.m_changeSetsMap.changeSets[1] + Map changes = history.m_changeSetsMap + System.err.println(changes) //System.err.println("added:\n\t" + changes.addedPaths) //System.err.println("removed:\n\t" + changes.removedPaths) + + //whelk.getStorage().insertNotice(untilVersion.versionID, USERID, changes) } } catch (Throwable e) { diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 68206bcfe5..ed12311484 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -16,12 +16,13 @@ public abstract class HouseKeeper { public abstract void trigger() public ZonedDateTime lastFailAt = null + public ZonedDateTime lastRunAt = null } @CompileStatic @Log public class WebInterface extends HttpServlet { - private static final long PERIODIC_TRIGGER_MS = 10 * 1000; + private static final long PERIODIC_TRIGGER_MS = 10 * 1000 private final Timer timer = new Timer("Housekeeper-timer", true) private List houseKeepers = [] @@ -35,6 +36,7 @@ public class WebInterface extends HttpServlet { timer.scheduleAtFixedRate({ try { hk.trigger() + hk.lastRunAt = ZonedDateTime.now() } catch (Throwable e) { log.error("Could not handle throwable in Housekeeper TimerTask.", e) hk.lastFailAt = ZonedDateTime.now() @@ -53,6 +55,10 @@ public class WebInterface extends HttpServlet { sb.append("--------------\n") for (HouseKeeper hk : houseKeepers) { sb.append(hk.getName() + "\n") + if (hk.lastRunAt) + sb.append("last run at: " + hk.lastRunAt + "\n") + else + sb.append("has never run\n") if (hk.lastFailAt) sb.append("last failed at: " + hk.lastFailAt + "\n") else diff --git a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql index 1de62d84c2..2651ef871a 100755 --- a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql +++ b/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql @@ -28,6 +28,7 @@ CREATE TABLE IF NOT EXISTS lddb__notices ( pk SERIAL PRIMARY KEY, versionid INTEGER, userid TEXT, + changes jsonb not null, handled BOOLEAN DEFAULT FALSE, created timestamp with time zone DEFAULT now() NOT NULL, diff --git a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy index 9ed401b0d8..03e0a29b52 100644 --- a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy +++ b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy @@ -817,8 +817,8 @@ class CrudSpec extends Specification { } storage.loadDocumentHistory(_) >> { [ - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", ""), - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", ""), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", "", 0), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", "", 0), ] } diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index cba3868c74..b124685cbc 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -158,7 +158,7 @@ class PostgreSQLComponent { """.stripIndent() private static final String GET_ALL_DOCUMENT_VERSIONS = """ - SELECT id, data, deleted, created, modified, changedBy, changedIn + SELECT id, data, deleted, created, modified, changedBy, changedIn, pk FROM lddb__versions WHERE id = ? ORDER BY GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) ASC @@ -399,6 +399,15 @@ class PostgreSQLComponent { JOIN lddb ON lddb__identifiers.id = lddb.id WHERE lddb__identifiers.iri = ? """.stripIndent() + private static final String INSERT_NOTICE = """ + INSERT INTO lddb__notices (versionid, userid, changes) + VALUES (?, ?, ?) + """.stripIndent() + + private static final String GET_NOTICES_FOR_USER = """ + SELECT * FROM lddb__notices WHERE userid = ? ORDER BY created ASC + """.stripIndent() + private HikariDataSource connectionPool private HikariDataSource outerConnectionPool @@ -1375,6 +1384,48 @@ class PostgreSQLComponent { storeCard(cardEntry) return cardEntry.getCard().data } + + Map getNoticesFor(String userid) { + return withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + ResultSet rs = null + try { + preparedStatement = connection.prepareStatement(GET_NOTICES_FOR_USER) + preparedStatement.setString(1, userid) + + rs = preparedStatement.executeQuery() + List results = [] + while(rs.next()) { + Map notice = [ + "versionID" : rs.getInt("versionid"), + "changes" : mapper.readValue(rs.getString("changes"), Map), + "handled" : rs.getBoolean("handled") + ] + results.add(notice) + } + return results + } finally { + close(rs, preparedStatement) + } + } + } + + boolean insertNotice(int versionID, String userID, Map changes) { + return withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + try { + preparedStatement = connection.prepareStatement(INSERT_NOTICE) + preparedStatement.setInt(1, versionID) + preparedStatement.setString(2, userID) + preparedStatement.setObject(3, mapper.writeValueAsString(changes), OTHER) + return (preparedStatement.executeUpdate() == 1) + } finally { + close(preparedStatement) + } + } + } void recalculateDependencies(Document doc) { withDbConnection { @@ -2364,7 +2415,7 @@ class PostgreSQLComponent { while (rs.next()) { def doc = assembleDocument(rs) doc.version = v++ - docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"))) + docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"), rs.getInt("pk"))) } } finally { close(rs, selectstmt) diff --git a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java index 9f9f68384f..c9665b030e 100644 --- a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java +++ b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java @@ -9,9 +9,11 @@ public class DocumentVersion { public Document doc; public String changedBy; public String changedIn; - public DocumentVersion(Document doc, String changedBy, String changedIn) { + public int versionID; + public DocumentVersion(Document doc, String changedBy, String changedIn, int versionID) { this.doc = doc; this.changedBy = changedBy; this.changedIn = changedIn; + this.versionID = versionID; } } diff --git a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy index 2ed0826f3c..455b28a1bc 100644 --- a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy +++ b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy @@ -688,7 +688,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) @@ -723,7 +723,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) @@ -758,7 +758,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) } def history = new History(versions, ld) @@ -776,7 +776,7 @@ class HistorySpec extends Specification { def ld = new JsonLd(JsonLdSpec.CONTEXT_DATA, display, JsonLdSpec.VOCAB_DATA) def v = versions.collect { data -> - new DocumentVersion(new Document(data), '', '') + new DocumentVersion(new Document(data), '', '', 0) } def history = new History(v, ld) From 124a15d066a4c676354f4d0fbba1b4852c7b07e3 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 28 Apr 2023 12:28:25 +0200 Subject: [PATCH 04/58] progress towards CXZ --- .../whelk/housekeeping/CXZGenerator.groovy | 35 ++++++++++-- .../component/PostgreSQLComponent.groovy | 53 +++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy index ee04ebdf20..c0d458643c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy @@ -1,5 +1,6 @@ package whelk.housekeeping +import whelk.Document import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log @@ -11,9 +12,8 @@ import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.Timestamp import java.time.Instant -import java.time.ZonedDateTime import java.time.temporal.ChronoUnit -import java.time.temporal.TemporalUnit +import static whelk.util.Jackson.mapper @CompileStatic @Log @@ -36,6 +36,26 @@ class CXZGenerator extends HouseKeeper { public void trigger() { + // Build a multi-map of library -> list of settings objects for that library's users + List allUserSettingStrings = whelk.getStorage().getAllUserData() + Map> libraryToUsers = new HashMap<>() + for (Map settings: allUserSettingStrings) { + settings?.requestedNotices.each { request -> + if (! request instanceof Map) + return + if (! request["library"]) + return + + String library = request["library"] + if (!libraryToUsers.containsKey(library)) + libraryToUsers.put(library, []) + List userSettingsForThisLib = libraryToUsers[library] + userSettingsForThisLib.add(settings) + } + } + + System.err.println("Libraries to all there users and settings:\n\t" + libraryToUsers) + Connection connection PreparedStatement statement ResultSet resultSet @@ -44,7 +64,7 @@ class CXZGenerator extends HouseKeeper { connection.setAutoCommit(false) try { - // First, determine the time interval of changes for which to generate notices. + // Determine the time interval of changes for which to generate notices. // This interval, should generally be: From the last generated notice until now. // However, if there are no previously generated notices (near enough in time), use // now - [some pre set value], to avoid scanning the whole catalog. @@ -60,7 +80,7 @@ class CXZGenerator extends HouseKeeper { Timestamp until = Timestamp.from(Instant.now()) // Then fetch all changes within that interval - sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth', 'hold') AND ( modified BETWEEN ? AND ? );"; + sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified BETWEEN ? AND ? );"; connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -99,6 +119,13 @@ class CXZGenerator extends HouseKeeper { //System.err.println("added:\n\t" + changes.addedPaths) //System.err.println("removed:\n\t" + changes.removedPaths) + // TODO: Om id är i 'auth' gör istället allt nedan för beroende (och embellished!) instanser + + List libraries = whelk.getStorage().getAllLibrariesHolding(id) + System.err.println(" heldBy: " + libraries) + + + //whelk.getStorage().insertNotice(untilVersion.versionID, USERID, changes) } diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index b124685cbc..b7b0d8ba01 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -383,6 +383,9 @@ class PostgreSQLComponent { private static final String GET_USER_DATA = "SELECT data FROM lddb__user_data WHERE id = ?" + private static final String GET_ALL_USER_DATA = + "SELECT id, data FROM lddb__user_data" + private static final String UPSERT_USER_DATA = """ INSERT INTO lddb__user_data (id, data, modified) VALUES (?, ?, ?) @@ -408,6 +411,11 @@ class PostgreSQLComponent { SELECT * FROM lddb__notices WHERE userid = ? ORDER BY created ASC """.stripIndent() + private static final String GET_ALL_LIBRARIES_HOLDING_ID = """ + SELECT l.data#>>'{@graph,1,heldBy,@id}' FROM lddb__dependencies d + LEFT JOIN lddb l ON d.id = l.id + WHERE d.dependsonid = ? AND d.relation = 'itemOf'""" + private HikariDataSource connectionPool private HikariDataSource outerConnectionPool @@ -1385,6 +1393,27 @@ class PostgreSQLComponent { return cardEntry.getCard().data } + List getAllLibrariesHolding(String id) { + return withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + ResultSet rs = null + try { + preparedStatement = connection.prepareStatement(GET_ALL_LIBRARIES_HOLDING_ID) + preparedStatement.setString(1, id) + + rs = preparedStatement.executeQuery() + List results = [] + while(rs.next()) { + results.add(rs.getString(1)) + } + return results + } finally { + close(rs, preparedStatement) + } + } + } + Map getNoticesFor(String userid) { return withDbConnection { Connection connection = getMyConnection() @@ -2615,6 +2644,30 @@ class PostgreSQLComponent { } } + /** + * Returns the user-data map for each user _with the user id_ also inserted into the map. + */ + List getAllUserData() { + return withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + ResultSet rs = null + List result = [] + try { + preparedStatement = connection.prepareStatement(GET_ALL_USER_DATA) + rs = preparedStatement.executeQuery() + while (rs.next()) { + Map userdata = mapper.readValue(rs.getString("data"), Map) + userdata.put("id", rs.getString("id")) + result.add(userdata) + } + return result + } finally { + close(rs, preparedStatement) + } + } + } + String getUserData(String id) { return withDbConnection { Connection connection = getMyConnection() From 844e4cdd63acbba03c5c5eae210bd04bbebe05c5 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 28 Apr 2023 14:01:44 +0200 Subject: [PATCH 05/58] refactor --- .../whelk/housekeeping/CXZGenerator.groovy | 109 ++++++++++-------- 1 file changed, 58 insertions(+), 51 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy index c0d458643c..504235c2ee 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy @@ -37,25 +37,25 @@ class CXZGenerator extends HouseKeeper { public void trigger() { // Build a multi-map of library -> list of settings objects for that library's users - List allUserSettingStrings = whelk.getStorage().getAllUserData() - Map> libraryToUsers = new HashMap<>() - for (Map settings: allUserSettingStrings) { - settings?.requestedNotices.each { request -> - if (! request instanceof Map) - return - if (! request["library"]) - return - - String library = request["library"] - if (!libraryToUsers.containsKey(library)) - libraryToUsers.put(library, []) - List userSettingsForThisLib = libraryToUsers[library] - userSettingsForThisLib.add(settings) + Map> libraryToUsers = new HashMap<>(); + { + List allUserSettingStrings = whelk.getStorage().getAllUserData() + for (Map settings : allUserSettingStrings) { + settings?.requestedNotices.each { request -> + if (!request instanceof Map) + return + if (!request["library"]) + return + + String library = request["library"] + if (!libraryToUsers.containsKey(library)) + libraryToUsers.put(library, []) + List userSettingsForThisLib = libraryToUsers[library] + userSettingsForThisLib.add(settings) + } } } - System.err.println("Libraries to all there users and settings:\n\t" + libraryToUsers) - Connection connection PreparedStatement statement ResultSet resultSet @@ -63,7 +63,6 @@ class CXZGenerator extends HouseKeeper { connection = whelk.getStorage().getOuterConnection() connection.setAutoCommit(false) try { - // Determine the time interval of changes for which to generate notices. // This interval, should generally be: From the last generated notice until now. // However, if there are no previously generated notices (near enough in time), use @@ -89,53 +88,61 @@ class CXZGenerator extends HouseKeeper { resultSet = statement.executeQuery() while (resultSet.next()) { String id = resultSet.getString("id") + generateNoticesForChange(id, libraryToUsers, from.toInstant()) + } - // "versions" come sorted by ascending modification time, so oldest version first. - // We want to pick the "from version" (the base for which this notice details changes) - // as the last saved version *before* the sought interval. - DocumentVersion fromVersion = null - List versions = whelk.getStorage().loadDocumentHistory(id) - for (DocumentVersion version : versions) { - if (version.doc.getModifiedTimestamp().isBefore(from.toInstant())) - fromVersion = version - } - if (fromVersion == null) - continue - - DocumentVersion untilVersion = versions.last() - if (untilVersion == fromVersion) - continue + } catch (Throwable e) { + status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() + throw e + } finally { + connection.close() + } + } - List relevantVersions = [] - relevantVersions.add(fromVersion) - relevantVersions.add(untilVersion) + private void generateNoticesForChange(String id, Map libraryToUsers, Instant since) { + // "versions" come sorted by ascending modification time, so oldest version first. + // We want to pick the "from version" (the base for which this notice details changes) + // as the last saved version *before* the sought interval. + DocumentVersion fromVersion = null + List versions = whelk.getStorage().loadDocumentHistory(id) + for (DocumentVersion version : versions) { + if (version.doc.getModifiedTimestamp().isBefore(since)) + fromVersion = version + } + if (fromVersion == null) + return - System.err.println("Was changed: " + id + " spans: " + fromVersion.doc.getModified() + " -> " + untilVersion.doc.getModified()) + DocumentVersion untilVersion = versions.last() + if (untilVersion == fromVersion) + return - History history = new History(relevantVersions, whelk.getJsonld()) + List relevantVersions = [] + relevantVersions.add(fromVersion) + relevantVersions.add(untilVersion) - Map changes = history.m_changeSetsMap - System.err.println(changes) - //System.err.println("added:\n\t" + changes.addedPaths) - //System.err.println("removed:\n\t" + changes.removedPaths) + System.err.println("Was changed: " + id + " spans: " + fromVersion.doc.getModified() + " -> " + untilVersion.doc.getModified()) - // TODO: Om id är i 'auth' gör istället allt nedan för beroende (och embellished!) instanser + History history = new History(relevantVersions, whelk.getJsonld()) - List libraries = whelk.getStorage().getAllLibrariesHolding(id) - System.err.println(" heldBy: " + libraries) + Map changes = history.m_changeSetsMap + System.err.println(changes) + //System.err.println("added:\n\t" + changes.addedPaths) + //System.err.println("removed:\n\t" + changes.removedPaths) + // TODO: Om id är i 'auth' gör istället allt nedan för beroende (och embellished!) instanser + List libraries = whelk.getStorage().getAllLibrariesHolding(id) - //whelk.getStorage().insertNotice(untilVersion.versionID, USERID, changes) + for (String library : libraries) { + List users = (List) libraryToUsers[library] + if (users) { + for (Map user : users) { + System.err.println("" + user["id"].toString() + " has requested updates for " + library) + } } - - } catch (Throwable e) { - status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() - throw e - } finally { - connection.close() } + //whelk.getStorage().insertNotice(untilVersion.versionID, USERID, changes) } From 9096d0fbd55c3dcc1b153a11953aac77a3436f2f Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 3 May 2023 13:06:49 +0200 Subject: [PATCH 06/58] Actually write notice. --- .../whelk/housekeeping/CXZGenerator.groovy | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy index 504235c2ee..defb3d935c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy @@ -78,7 +78,7 @@ class CXZGenerator extends HouseKeeper { } Timestamp until = Timestamp.from(Instant.now()) - // Then fetch all changes within that interval + // Then fetch all changed IDs within that interval sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified BETWEEN ? AND ? );"; connection.setAutoCommit(false) statement = connection.prepareStatement(sql) @@ -137,12 +137,34 @@ class CXZGenerator extends HouseKeeper { List users = (List) libraryToUsers[library] if (users) { for (Map user : users) { - System.err.println("" + user["id"].toString() + " has requested updates for " + library) + + /* + user is a map looking something like this: + { + "id": "sldknfslkdnsdlkgnsdkjgnb" + "requestedNotices": [ + {"library": "https://libris.kb.se/library/Utb1", "trigger": ["@graph", 1, "contribution", "@type=PrimaryContribution", "agent"]}, + {"library": "https://libris.kb.se/library/Utb2", "trigger": ["@graph", 1, "instanceOf", "hasTitle", "*", "mainTitle"]} + ] + } + */ + //System.err.println("" + user["id"].toString() + " has requested updates for " + library) + + if (changeMatchesAnyTrigger(fromVersion, untilVersion, user)) { + whelk.getStorage().insertNotice(untilVersion.versionID, user["id"].toString(), changes) + System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) + } } } } + } - //whelk.getStorage().insertNotice(untilVersion.versionID, USERID, changes) + /** + * Parameters are the two relevant versions (before and after), and user is + * the user-data map for a user (which includes their selection of triggers) + */ + private boolean changeMatchesAnyTrigger(DocumentVersion fromVersion, DocumentVersion untilVersion, Map user) { + return true } From 33a4727218659be17206f2a9c115bf8dcfa04c8c Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 3 May 2023 16:26:39 +0200 Subject: [PATCH 07/58] Change to consistent usage of the name 'notifications' instead of CXZ or otherwise. --- ...rator.groovy => NotificationGenerator.groovy} | 11 +++++------ .../whelk/housekeeping/WebInterface.groovy | 2 +- ...ql => 00000021-add-notifications-table.plsql} | 6 +++--- .../groovy/whelk/rest/api/UserDataAPI.groovy | 4 ++-- rest/src/main/webapp/WEB-INF/web.xml | 14 ++++++++++++++ .../whelk/component/PostgreSQLComponent.groovy | 16 ++++++++-------- 6 files changed, 33 insertions(+), 20 deletions(-) rename housekeeping/src/main/groovy/whelk/housekeeping/{CXZGenerator.groovy => NotificationGenerator.groovy} (96%) rename librisxl-tools/postgresql/migrations/{00000021-add-notice-table.plsql => 00000021-add-notifications-table.plsql} (83%) mode change 100755 => 100644 diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy similarity index 96% rename from housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy rename to housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index defb3d935c..5dfce7b44e 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/CXZGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -1,6 +1,6 @@ package whelk.housekeeping -import whelk.Document + import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log @@ -13,16 +13,15 @@ import java.sql.ResultSet import java.sql.Timestamp import java.time.Instant import java.time.temporal.ChronoUnit -import static whelk.util.Jackson.mapper @CompileStatic @Log -class CXZGenerator extends HouseKeeper { +class NotificationGenerator extends HouseKeeper { private String status = "OK" private Whelk whelk - public CXZGenerator(Whelk whelk) { + public NotificationGenerator(Whelk whelk) { this.whelk = whelk } @@ -67,7 +66,7 @@ class CXZGenerator extends HouseKeeper { // This interval, should generally be: From the last generated notice until now. // However, if there are no previously generated notices (near enough in time), use // now - [some pre set value], to avoid scanning the whole catalog. - String sql = "SELECT MAX(created) FROM lddb__notices;" + String sql = "SELECT MAX(created) FROM lddb__notifications;" statement = connection.prepareStatement(sql) resultSet = statement.executeQuery() Timestamp from = Timestamp.from(Instant.now().minus(2, ChronoUnit.DAYS)) @@ -151,7 +150,7 @@ class CXZGenerator extends HouseKeeper { //System.err.println("" + user["id"].toString() + " has requested updates for " + library) if (changeMatchesAnyTrigger(fromVersion, untilVersion, user)) { - whelk.getStorage().insertNotice(untilVersion.versionID, user["id"].toString(), changes) + whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), changes) System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) } } diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index ed12311484..a9e4268896 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -30,7 +30,7 @@ public class WebInterface extends HttpServlet { Whelk whelk = Whelk.createLoadedCoreWhelk() houseKeepers = [] - houseKeepers.add(new CXZGenerator(whelk)) + houseKeepers.add(new NotificationGenerator(whelk)) for (HouseKeeper hk : houseKeepers) { timer.scheduleAtFixedRate({ diff --git a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql old mode 100755 new mode 100644 similarity index 83% rename from librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql rename to librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql index 2651ef871a..332f69682d --- a/librisxl-tools/postgresql/migrations/00000021-add-notice-table.plsql +++ b/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql @@ -24,7 +24,7 @@ UPDATE lddb__schema SET version = new_version; -- ACTUAL SCHEMA CHANGES HERE: ALTER TABLE lddb__versions ADD UNIQUE (pk); -CREATE TABLE IF NOT EXISTS lddb__notices ( +CREATE TABLE IF NOT EXISTS lddb__notifications ( pk SERIAL PRIMARY KEY, versionid INTEGER, userid TEXT, @@ -35,8 +35,8 @@ CREATE TABLE IF NOT EXISTS lddb__notices ( CONSTRAINT version_fk FOREIGN KEY (versionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, CONSTRAINT user_fk FOREIGN KEY (userid) REFERENCES lddb__user_data(id) ON DELETE CASCADE ); -CREATE INDEX idx_notices_user ON lddb__notices USING BTREE (userid); -CREATE INDEX idx_notices_created ON lddb__notices USING BTREE (created); +CREATE INDEX idx_notifications_user ON lddb__notifications USING BTREE (userid); +CREATE INDEX idx_notifications_created ON lddb__notifications USING BTREE (created); END$$; diff --git a/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy b/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy index f14181b754..f041a9a632 100644 --- a/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy +++ b/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy @@ -16,7 +16,7 @@ import java.util.stream.Collectors class UserDataAPI extends HttpServlet { private Whelk whelk private static final int POST_MAX_SIZE = 1000000 - private static final String ID_HASH_FUNCTION = "SHA-256" + static final String ID_HASH_FUNCTION = "SHA-256" UserDataAPI() { } @@ -125,7 +125,7 @@ class UserDataAPI extends HttpServlet { return data } - private static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { + static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { if (!userInfo) { log.info("User authentication failed") response.sendError(HttpServletResponse.SC_FORBIDDEN, "User authentication failed") diff --git a/rest/src/main/webapp/WEB-INF/web.xml b/rest/src/main/webapp/WEB-INF/web.xml index 345a555382..5abdd7fa2f 100644 --- a/rest/src/main/webapp/WEB-INF/web.xml +++ b/rest/src/main/webapp/WEB-INF/web.xml @@ -98,6 +98,10 @@ TransliterationAPI whelk.rest.api.TransliterationAPI + + NotificationsAPI + whelk.rest.api.NotificationsAPI + @@ -106,6 +110,11 @@ 1 + + NotificationsAPI + /_notifications/* + + UserDataAPI /_userdata/* @@ -181,6 +190,11 @@ /_userdata/* + + AuthenticationFilterGet + /_notifications/* + + AuthenticationFilterGet /_remotesearch diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index b7b0d8ba01..c2ef6c219c 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -402,13 +402,13 @@ class PostgreSQLComponent { JOIN lddb ON lddb__identifiers.id = lddb.id WHERE lddb__identifiers.iri = ? """.stripIndent() - private static final String INSERT_NOTICE = """ - INSERT INTO lddb__notices (versionid, userid, changes) + private static final String INSERT_NOTIFICATION = """ + INSERT INTO lddb__notifications (versionid, userid, changes) VALUES (?, ?, ?) """.stripIndent() - private static final String GET_NOTICES_FOR_USER = """ - SELECT * FROM lddb__notices WHERE userid = ? ORDER BY created ASC + private static final String GET_NOTICFICATIONS_FOR_USER = """ + SELECT * FROM lddb__notifications WHERE userid = ? ORDER BY created ASC """.stripIndent() private static final String GET_ALL_LIBRARIES_HOLDING_ID = """ @@ -1414,13 +1414,13 @@ class PostgreSQLComponent { } } - Map getNoticesFor(String userid) { + List getNotificationsFor(String userid) { return withDbConnection { Connection connection = getMyConnection() PreparedStatement preparedStatement = null ResultSet rs = null try { - preparedStatement = connection.prepareStatement(GET_NOTICES_FOR_USER) + preparedStatement = connection.prepareStatement(GET_NOTICFICATIONS_FOR_USER) preparedStatement.setString(1, userid) rs = preparedStatement.executeQuery() @@ -1440,12 +1440,12 @@ class PostgreSQLComponent { } } - boolean insertNotice(int versionID, String userID, Map changes) { + boolean insertNotification(int versionID, String userID, Map changes) { return withDbConnection { Connection connection = getMyConnection() PreparedStatement preparedStatement = null try { - preparedStatement = connection.prepareStatement(INSERT_NOTICE) + preparedStatement = connection.prepareStatement(INSERT_NOTIFICATION) preparedStatement.setInt(1, versionID) preparedStatement.setString(2, userID) preparedStatement.setObject(3, mapper.writeValueAsString(changes), OTHER) From 2700e51f0ea91a2b6f29ca85e3dd268f363a5a30 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 3 May 2023 16:29:55 +0200 Subject: [PATCH 08/58] Add missing file --- .../whelk/rest/api/NotificationsAPI.groovy | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy diff --git a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy new file mode 100644 index 0000000000..1945df6d12 --- /dev/null +++ b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy @@ -0,0 +1,35 @@ +package whelk.rest.api + +import groovy.util.logging.Log4j2 as Log +import whelk.Whelk +import whelk.util.WhelkFactory + +import javax.servlet.http.HttpServlet +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +@Log +class NotificationsAPI extends HttpServlet { + + private Whelk whelk + + @Override + void init() { + log.info("Starting User Data API") + if (!whelk) { + whelk = WhelkFactory.getSingletonWhelk() + } + } + + @Override + void doGet(HttpServletRequest request, HttpServletResponse response) { + Map userInfo = request.getAttribute("user") + if (!UserDataAPI.isValidUserWithPermission(request, response, userInfo)) + return + + String id = "${userInfo.id}".digest(UserDataAPI.ID_HASH_FUNCTION) + List data = whelk.getStorage().getNotificationsFor(id) + + HttpTools.sendResponse(response, ["THE LIST": data], "application/json") + } +} From 5474eb3061fc969c550a84eb794884130b927cde Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 4 May 2023 14:59:31 +0200 Subject: [PATCH 09/58] Add the ability to flip the handled marker on notifications by POST to them. --- .../whelk/rest/api/NotificationsAPI.groovy | 35 +++++++++++++++++-- .../groovy/whelk/rest/api/UserDataAPI.groovy | 2 +- .../component/PostgreSQLComponent.groovy | 20 +++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy index 1945df6d12..9ec1bbd2d5 100644 --- a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy +++ b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy @@ -15,7 +15,7 @@ class NotificationsAPI extends HttpServlet { @Override void init() { - log.info("Starting User Data API") + log.info("Starting Notifications API") if (!whelk) { whelk = WhelkFactory.getSingletonWhelk() } @@ -24,12 +24,41 @@ class NotificationsAPI extends HttpServlet { @Override void doGet(HttpServletRequest request, HttpServletResponse response) { Map userInfo = request.getAttribute("user") - if (!UserDataAPI.isValidUserWithPermission(request, response, userInfo)) + if (!isValidUserWithPermission(request, response, userInfo)) return String id = "${userInfo.id}".digest(UserDataAPI.ID_HASH_FUNCTION) List data = whelk.getStorage().getNotificationsFor(id) - HttpTools.sendResponse(response, ["THE LIST": data], "application/json") + HttpTools.sendResponse(response, ["notifications": data], "application/json") + } + + @Override + void doPost(HttpServletRequest request, HttpServletResponse response) { + log.info("Handling POST request for ${request.pathInfo}") + + Map userInfo = request.getAttribute("user") + if (!isValidUserWithPermission(request, response, userInfo)) + return + + String userID = "${userInfo.id}".digest(UserDataAPI.ID_HASH_FUNCTION) + String notificationID = request.getPathInfo().replace("/", "") + whelk.getStorage().flipNotificationHandled(userID, Integer.parseInt(notificationID)) + } + + static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { + if (!userInfo) { + log.info("User authentication failed") + response.sendError(HttpServletResponse.SC_FORBIDDEN, "User authentication failed") + return false + } + + if (!userInfo.containsKey("id")) { + log.info("User check failed: 'id' missing in user") + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Key 'id' missing in user info") + return false + } + + return true } } diff --git a/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy b/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy index f041a9a632..7dd0c0165a 100644 --- a/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy +++ b/rest/src/main/groovy/whelk/rest/api/UserDataAPI.groovy @@ -125,7 +125,7 @@ class UserDataAPI extends HttpServlet { return data } - static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { + private static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { if (!userInfo) { log.info("User authentication failed") response.sendError(HttpServletResponse.SC_FORBIDDEN, "User authentication failed") diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index c2ef6c219c..89877ce3c6 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -411,6 +411,10 @@ class PostgreSQLComponent { SELECT * FROM lddb__notifications WHERE userid = ? ORDER BY created ASC """.stripIndent() + private static final String FLIP_NOTIFICATION_HANDLED = """ + UPDATE lddb__notifications SET handled = NOT handled WHERE userid = ? AND pk = ? + """.stripIndent() + private static final String GET_ALL_LIBRARIES_HOLDING_ID = """ SELECT l.data#>>'{@graph,1,heldBy,@id}' FROM lddb__dependencies d LEFT JOIN lddb l ON d.id = l.id @@ -1427,6 +1431,7 @@ class PostgreSQLComponent { List results = [] while(rs.next()) { Map notice = [ + "notificationID": rs.getInt("pk"), "versionID" : rs.getInt("versionid"), "changes" : mapper.readValue(rs.getString("changes"), Map), "handled" : rs.getBoolean("handled") @@ -1455,6 +1460,21 @@ class PostgreSQLComponent { } } } + + boolean flipNotificationHandled(String userID, int notificationID) { + return withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + try { + preparedStatement = connection.prepareStatement(FLIP_NOTIFICATION_HANDLED) + preparedStatement.setString(1, userID) + preparedStatement.setInt(2, notificationID) + return (preparedStatement.executeUpdate() == 1) + } finally { + close(preparedStatement) + } + } + } void recalculateDependencies(Document doc) { withDbConnection { From 41ee45ef32b2c6010487808f7fe85732b421a41d Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 5 May 2023 10:26:01 +0200 Subject: [PATCH 10/58] Notifications are for instances, but can be a result of changes in a dependency. --- .../housekeeping/NotificationGenerator.groovy | 106 +++++------------- .../component/PostgreSQLComponent.groovy | 29 +++++ 2 files changed, 60 insertions(+), 75 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 5dfce7b44e..4b1650da6c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -1,11 +1,9 @@ package whelk.housekeeping - import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log import whelk.history.DocumentVersion -import whelk.history.History import java.sql.Connection import java.sql.PreparedStatement @@ -26,7 +24,7 @@ class NotificationGenerator extends HouseKeeper { } public String getName() { - return "CXZ notifier generator" + return "Notifications generator" } public String getStatusDescription() { @@ -87,7 +85,7 @@ class NotificationGenerator extends HouseKeeper { resultSet = statement.executeQuery() while (resultSet.next()) { String id = resultSet.getString("id") - generateNoticesForChange(id, libraryToUsers, from.toInstant()) + generateNoticesForChangedID(id, libraryToUsers, from.toInstant()) } } catch (Throwable e) { @@ -98,7 +96,7 @@ class NotificationGenerator extends HouseKeeper { } } - private void generateNoticesForChange(String id, Map libraryToUsers, Instant since) { + private void generateNoticesForChangedID(String id, Map libraryToUsers, Instant since) { // "versions" come sorted by ascending modification time, so oldest version first. // We want to pick the "from version" (the base for which this notice details changes) // as the last saved version *before* the sought interval. @@ -115,23 +113,24 @@ class NotificationGenerator extends HouseKeeper { if (untilVersion == fromVersion) return - List relevantVersions = [] - relevantVersions.add(fromVersion) - relevantVersions.add(untilVersion) - - System.err.println("Was changed: " + id + " spans: " + fromVersion.doc.getModified() + " -> " + untilVersion.doc.getModified()) - - History history = new History(relevantVersions, whelk.getJsonld()) - - Map changes = history.m_changeSetsMap - System.err.println(changes) - //System.err.println("added:\n\t" + changes.addedPaths) - //System.err.println("removed:\n\t" + changes.removedPaths) + List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) + dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! + dependers.each { + String dependerID = it[0] + String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) + if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { + generateNoticesForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion) + } + } - // TODO: Om id är i 'auth' gör istället allt nedan för beroende (och embellished!) instanser + } + /** + * Generate notice for a bibliographic instance. Beware: fromVersion and untilVersion may not be + * _of this document_ (id), but rather of a document this instance depends on! + */ + private void generateNoticesForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, DocumentVersion untilVersion) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) - for (String library : libraries) { List users = (List) libraryToUsers[library] if (users) { @@ -149,6 +148,19 @@ class NotificationGenerator extends HouseKeeper { */ //System.err.println("" + user["id"].toString() + " has requested updates for " + library) + /*List relevantVersions = [] + relevantVersions.add(fromVersion) + relevantVersions.add(untilVersion)*/ + + /*System.err.println("Was changed: " + id + " spans: " + fromVersion.doc.getModified() + " -> " + untilVersion.doc.getModified()) + + History history = new History(relevantVersions, whelk.getJsonld()) + + Map changes = history.m_changeSetsMap + System.err.println(changes)*/ + + Map changes = ["STILL":"TODO"] + if (changeMatchesAnyTrigger(fromVersion, untilVersion, user)) { whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), changes) System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) @@ -166,60 +178,4 @@ class NotificationGenerator extends HouseKeeper { return true } - - //zonedFrom = ZonedDateTime.parse(from); - //zonedUntil = ZonedDateTime.parse(until); - /*public void generate(ZonedDateTime zonedFrom, ZonedDateTime zonedUntil) throws SQLException { - - Connection connection; - PreparedStatement statement; - ResultSet resultSet; - - connection = whelk.getStorage().getOuterConnection(); - try { - String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth', 'hold') AND ( modified BETWEEN ? AND ? );"; - connection.setAutoCommit(false); - statement = connection.prepareStatement(sql); - statement.setTimestamp(1, new Timestamp(zonedFrom.toInstant().getEpochSecond() * 1000L)); - statement.setTimestamp(2, new Timestamp(zonedUntil.toInstant().getEpochSecond() * 1000L)); - statement.setFetchSize(512); - resultSet = statement.executeQuery(); - } catch (Throwable e) { - connection.close(); - throw e; - } - }*/ - - -} - - -/* - if (fromVersion < 0 || fromVersion >= untilVersion) - return // error out - - // We don't need all versions, just specifically the starting and end point of the - // sought interval. - List versions = whelk.getStorage().loadDocumentHistory(id) - List relevantVersions = [] - relevantVersions.add(versions.get(fromVersion)) - relevantVersions.add(versions.get(untilVersion)) - - History history = new History(relevantVersions, whelk.getJsonld()) - Map changes = history.m_changeSetsMap.changeSets[1] - System.err.println("added:\n\t" + changes.addedPaths) - System.err.println("removed:\n\t" + changes.removedPaths) - */ - - -/* -void doGet(HttpServletRequest request, HttpServletResponse response) { -{ - Map userInfo = request.getAttribute("user") - if (!isValidUserWithPermission(request, response, userInfo)) - return - - String id = "${userInfo.id}".digest(ID_HASH_FUNCTION) - String data = whelk.getUserData(id) ?: upgradeOldEmailBasedEntry(userInfo) ?: "{}" } - */ diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 89877ce3c6..63f43747d6 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -311,6 +311,9 @@ class PostgreSQLComponent { private static final String GET_COLLECTION_BY_SYSTEM_ID = "SELECT collection FROM lddb where id = ?" + private static final String GET_MAINENTITY_TYPE_BY_SYSTEM_ID = + "SELECT data#>>'{@graph,1,@type}' FROM lddb WHERE id = ?" + /** This query does the same as LOAD_COLLECTIONS = "SELECT DISTINCT collection FROM lddb" but much faster because postgres does not yet have 'loose indexscan' aka 'index skip scan' https://wiki.postgresql.org/wiki/Loose_indexscan' */ @@ -1936,6 +1939,32 @@ class PostgreSQLComponent { } } + String getMainEntityTypeBySystemID(String id) { + return withDbConnection { + Connection connection = getMyConnection() + return getMainEntityTypeBySystemID(id, connection) + } + } + + String getMainEntityTypeBySystemID(String id, Connection connection) { + PreparedStatement selectStatement = null + ResultSet resultSet = null + + try { + selectStatement = connection.prepareStatement(GET_MAINENTITY_TYPE_BY_SYSTEM_ID) + selectStatement.setString(1, id) + resultSet = selectStatement.executeQuery() + + if (resultSet.next()) { + return resultSet.getString(1) + } + return null + } + finally { + close(resultSet, selectStatement) + } + } + Document load(String id) { return load(id, null) } From 082999f9e25fc05984923edc2778799747de2e8f Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 5 May 2023 14:31:07 +0200 Subject: [PATCH 11/58] Clean up CXZ a bit. --- .../housekeeping/NotificationGenerator.groovy | 108 ++++++++++++------ .../whelk/housekeeping/WebInterface.groovy | 2 - .../whelk/rest/api/NotificationsAPI.groovy | 2 + .../component/PostgreSQLComponent.groovy | 15 ++- 4 files changed, 88 insertions(+), 39 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 4b1650da6c..91765045ae 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -16,6 +16,7 @@ import java.time.temporal.ChronoUnit @Log class NotificationGenerator extends HouseKeeper { + private static final int DAYS_TO_KEEP_NOTIFICATIONS = 10 private String status = "OK" private Whelk whelk @@ -38,7 +39,7 @@ class NotificationGenerator extends HouseKeeper { { List allUserSettingStrings = whelk.getStorage().getAllUserData() for (Map settings : allUserSettingStrings) { - settings?.requestedNotices.each { request -> + settings?.requestedNotifications.each { request -> if (!request instanceof Map) return if (!request["library"]) @@ -67,7 +68,7 @@ class NotificationGenerator extends HouseKeeper { String sql = "SELECT MAX(created) FROM lddb__notifications;" statement = connection.prepareStatement(sql) resultSet = statement.executeQuery() - Timestamp from = Timestamp.from(Instant.now().minus(2, ChronoUnit.DAYS)) + Timestamp from = Timestamp.from(Instant.now().minus(DAYS_TO_KEEP_NOTIFICATIONS, ChronoUnit.DAYS)) if (resultSet.next()) { Timestamp lastCreated = resultSet.getTimestamp(1) if (lastCreated && lastCreated.after(from)) @@ -76,7 +77,7 @@ class NotificationGenerator extends HouseKeeper { Timestamp until = Timestamp.from(Instant.now()) // Then fetch all changed IDs within that interval - sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified BETWEEN ? AND ? );"; + sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified BETWEEN ? AND ? );" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -85,9 +86,15 @@ class NotificationGenerator extends HouseKeeper { resultSet = statement.executeQuery() while (resultSet.next()) { String id = resultSet.getString("id") - generateNoticesForChangedID(id, libraryToUsers, from.toInstant()) + generateNotificationsForChangedID(id, libraryToUsers, from.toInstant()) } + // Finally, clean out any notifications that are too old + sql = "DELETE FROM lddb__notifications WHERE created < ?" + statement = connection.prepareStatement(sql) + statement.setTimestamp(1, Timestamp.from(Instant.now().minus(DAYS_TO_KEEP_NOTIFICATIONS, ChronoUnit.DAYS))) + statement.executeUpdate() + } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() throw e @@ -96,7 +103,7 @@ class NotificationGenerator extends HouseKeeper { } } - private void generateNoticesForChangedID(String id, Map libraryToUsers, Instant since) { + private void generateNotificationsForChangedID(String id, Map libraryToUsers, Instant since) { // "versions" come sorted by ascending modification time, so oldest version first. // We want to pick the "from version" (the base for which this notice details changes) // as the last saved version *before* the sought interval. @@ -119,7 +126,7 @@ class NotificationGenerator extends HouseKeeper { String dependerID = it[0] String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { - generateNoticesForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion) + generateNotificationsForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion) } } @@ -129,7 +136,7 @@ class NotificationGenerator extends HouseKeeper { * Generate notice for a bibliographic instance. Beware: fromVersion and untilVersion may not be * _of this document_ (id), but rather of a document this instance depends on! */ - private void generateNoticesForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, DocumentVersion untilVersion) { + private void generateNotificationsForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, DocumentVersion untilVersion) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { List users = (List) libraryToUsers[library] @@ -137,45 +144,82 @@ class NotificationGenerator extends HouseKeeper { for (Map user : users) { /* - user is a map looking something like this: + 'user' is now a map looking something like this: { "id": "sldknfslkdnsdlkgnsdkjgnb" - "requestedNotices": [ - {"library": "https://libris.kb.se/library/Utb1", "trigger": ["@graph", 1, "contribution", "@type=PrimaryContribution", "agent"]}, - {"library": "https://libris.kb.se/library/Utb2", "trigger": ["@graph", 1, "instanceOf", "hasTitle", "*", "mainTitle"]} - ] - } - */ - //System.err.println("" + user["id"].toString() + " has requested updates for " + library) + "requestedNotifications": [ + { + "library": "https://libris.kb.se/library/Utb1", + "triggers": [ + "https://id.kb.se/notificationtriggers/sab", + "https://id.kb.se/notificationtriggers/primarycontribution" + ] + } + ] + }*/ + + List triggered = changeMatchesAnyTrigger(fromVersion, untilVersion, user, library) + if (triggered) { + whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), ["triggered" : triggered]) + System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) + } + } + } + } + } + + /** + * '*version' parameters are the two relevant versions (before and after). + * 'user' is the user-data map for a user (which includes their selection of triggers). + * 'library' is a library ("sigel") holding the instance in question. + * + * This function answers the question: Has 'user' requested to be notified of the change between + * 'fromVersion' and 'untilVersion' for instances held by 'library'? + * + * Returns the URIs of all triggered rules/triggers. + */ + private static List changeMatchesAnyTrigger(DocumentVersion fromVersion, DocumentVersion untilVersion, Map user, String library) { - /*List relevantVersions = [] - relevantVersions.add(fromVersion) - relevantVersions.add(untilVersion)*/ + List triggeredTriggers = [] - /*System.err.println("Was changed: " + id + " spans: " + fromVersion.doc.getModified() + " -> " + untilVersion.doc.getModified()) + user.requestedNotifications.each { request -> - History history = new History(relevantVersions, whelk.getJsonld()) + // This stuff (the request) comes from a user, so we must be super paranoid about it being correctly formed. - Map changes = history.m_changeSetsMap - System.err.println(changes)*/ + if (! request instanceof Map) + return - Map changes = ["STILL":"TODO"] + if (! request["library"] instanceof String) + return - if (changeMatchesAnyTrigger(fromVersion, untilVersion, user)) { - whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), changes) - System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) - } - } + if (request["library"] != library) + return + + if (! request["triggers"] instanceof List) + return + + for (Object triggerObject : request["triggers"]) { + if (! triggerObject instanceof String) + return + String triggerUri = (String) triggerObject + if (triggerIsTriggered(fromVersion, untilVersion, triggerUri)) + triggeredTriggers.add(triggerUri) } } + + return triggeredTriggers } /** - * Parameters are the two relevant versions (before and after), and user is - * the user-data map for a user (which includes their selection of triggers) + * Do the changes between 'fromVersion' and 'untilVersion' qualify to trigger 'triggerUri' ? */ - private boolean changeMatchesAnyTrigger(DocumentVersion fromVersion, DocumentVersion untilVersion, Map user) { - return true + private static boolean triggerIsTriggered(DocumentVersion fromVersion, DocumentVersion untilVersion, String triggerUri) { + switch (triggerUri) { + case "https://id.kb.se/notificationtriggers/primarycontribution": + return true + break + } + return false } } diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index a9e4268896..4040173f6b 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -67,8 +67,6 @@ public class WebInterface extends HttpServlet { sb.append(hk.statusDescription+"\n") sb.append("--------------\n") } - //res.sendStatus(HttpServletResponse.SC_ACCEPTED, sb.toString()) - //res.sendError(HttpServletResponse.SC_BAD_REQUEST, "CHYIIL.. DUDE"); res.setStatus(HttpServletResponse.SC_OK) res.setContentType("text/plain") res.getOutputStream().print(sb.toString()) diff --git a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy index 9ec1bbd2d5..2d7e99453d 100644 --- a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy +++ b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy @@ -44,6 +44,8 @@ class NotificationsAPI extends HttpServlet { String userID = "${userInfo.id}".digest(UserDataAPI.ID_HASH_FUNCTION) String notificationID = request.getPathInfo().replace("/", "") whelk.getStorage().flipNotificationHandled(userID, Integer.parseInt(notificationID)) + + response.setStatus(HttpServletResponse.SC_OK) } static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 63f43747d6..1458ebc9cb 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -411,7 +411,12 @@ class PostgreSQLComponent { """.stripIndent() private static final String GET_NOTICFICATIONS_FOR_USER = """ - SELECT * FROM lddb__notifications WHERE userid = ? ORDER BY created ASC + SELECT n.pk, n.changes, n.handled, v.data#>>'{@graph,1,@id}' thingid + FROM lddb__notifications n + LEFT JOIN lddb__versions v + ON n.versionid = v.pk + WHERE userid = ? + ORDER BY n.created ASC """.stripIndent() private static final String FLIP_NOTIFICATION_HANDLED = """ @@ -1433,13 +1438,13 @@ class PostgreSQLComponent { rs = preparedStatement.executeQuery() List results = [] while(rs.next()) { - Map notice = [ + Map notification = [ "notificationID": rs.getInt("pk"), - "versionID" : rs.getInt("versionid"), - "changes" : mapper.readValue(rs.getString("changes"), Map), + "mainEntityID" : rs.getString("thingid"), + "reason" : mapper.readValue(rs.getString("changes"), Map), "handled" : rs.getBoolean("handled") ] - results.add(notice) + results.add(notification) } return results } finally { From f15f3571ec58258747980729d1418b8ae673216c Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 8 May 2023 12:53:06 +0200 Subject: [PATCH 12/58] Progress towards CXZ notices, independant of how the data is split between records. --- .../housekeeping/NotificationGenerator.groovy | 85 +++++++++++++++++-- whelk-core/src/main/groovy/whelk/Whelk.groovy | 5 +- .../component/PostgreSQLComponent.groovy | 77 ++++++++++++++--- .../groovy/whelk/history/DocumentVersion.java | 7 +- 4 files changed, 156 insertions(+), 18 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 91765045ae..9ef3e583a2 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -1,5 +1,6 @@ package whelk.housekeeping +import whelk.Document import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log @@ -158,7 +159,7 @@ class NotificationGenerator extends HouseKeeper { ] }*/ - List triggered = changeMatchesAnyTrigger(fromVersion, untilVersion, user, library) + List triggered = changeMatchesAnyTrigger(id, fromVersion, untilVersion, user, library) if (triggered) { whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), ["triggered" : triggered]) System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) @@ -178,7 +179,7 @@ class NotificationGenerator extends HouseKeeper { * * Returns the URIs of all triggered rules/triggers. */ - private static List changeMatchesAnyTrigger(DocumentVersion fromVersion, DocumentVersion untilVersion, Map user, String library) { + private List changeMatchesAnyTrigger(String instanceId, DocumentVersion fromVersion, DocumentVersion untilVersion, Map user, String library) { List triggeredTriggers = [] @@ -202,7 +203,7 @@ class NotificationGenerator extends HouseKeeper { if (! triggerObject instanceof String) return String triggerUri = (String) triggerObject - if (triggerIsTriggered(fromVersion, untilVersion, triggerUri)) + if (triggerIsTriggered(instanceId, fromVersion, untilVersion, triggerUri)) triggeredTriggers.add(triggerUri) } } @@ -211,15 +212,89 @@ class NotificationGenerator extends HouseKeeper { } /** - * Do the changes between 'fromVersion' and 'untilVersion' qualify to trigger 'triggerUri' ? + * Do the changes between 'fromVersion' and 'untilVersion' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? */ - private static boolean triggerIsTriggered(DocumentVersion fromVersion, DocumentVersion untilVersion, String triggerUri) { + private boolean triggerIsTriggered(String instanceId, DocumentVersion fromVersion, DocumentVersion untilVersion, String triggerUri) { switch (triggerUri) { case "https://id.kb.se/notificationtriggers/primarycontribution": + //whelk.getStorage().load + //whelk.embellish() + //fromVersion. + //Document from = fromVersion.doc.clone() + //Document until = untilVersion.doc.clone() + + Document affectedInstance = whelk.getStorage().loadAsOf(instanceId, fromVersion.versionWriteTime) + + // If a depender is created after a dependency, it will ofc not have existed at the original writing time + // of the dependency, if so, simply load the first available version of the depender. + if (affectedInstance == null) + affectedInstance = whelk.getStorage().load(instanceId, "0") + + //System.err.println("**** Loaded historic version of INSTANCE: " + affectedInstance.getDataAsString()) + + historicEmbellish(affectedInstance, ["instanceOf", "contribution", "agent"], fromVersion.versionWriteTime.toInstant()) return true break } return false } + /** + * This is a simplified/specialized from of 'embellish', for historic data and using only select properties. + * The full general embellish code can not help us here, because it is based on the idea of cached cards, + * which can (and must!) only cache the latest/current data for each card, which isn't what we need here + * (we need to embellish older historic data). + */ + private historicEmbellish(Document doc, List properties, Instant asOf) { + List graphList = doc.data["@graph"] + + System.err.println("**** WILL NOW SCAN FOR LINKS:\n\t" + doc.getDataAsString()) + + Set uris = findLinkedURIs(graphList, properties) + + System.err.println("\tFound links (with chosen properties): " + uris) + + Map linkedDocumentsByUri = whelk.bulkLoad(uris, asOf) + + //System.err.println("Was able to fetch historic data for: " + linkedDocumentsByUri.keySet()) + } + + private Set findLinkedURIs(Object node, List properties) { + Set uris = [] + if (node instanceof List) { + for (Object element : node) { + uris.addAll(findLinkedURIs(element, properties)) + } + } + else if (node instanceof Map) { + for (String key : node.keySet()) { + if (properties.contains(key)) { + uris.addAll(getLinkIfAny(node[key])) + } + uris.addAll(findLinkedURIs(node[key], properties)) + } + } + return uris + } + + private List getLinkIfAny(Object node) { + System.err.println("\tCHECK getLinkIfAny with " + node) + List uris = [] + if (node instanceof Map) { + if (node.containsKey("@id")) { + uris.add((String) node["@id"]) + } + } + if (node instanceof List) { + for (Object element : node) { + if (element instanceof Map) { + if (element.containsKey("@id")) { + uris.add((String) element["@id"]) + } + } + } + } + return uris + } + } diff --git a/whelk-core/src/main/groovy/whelk/Whelk.groovy b/whelk-core/src/main/groovy/whelk/Whelk.groovy index 51f08c9095..3530959bbd 100644 --- a/whelk-core/src/main/groovy/whelk/Whelk.groovy +++ b/whelk-core/src/main/groovy/whelk/Whelk.groovy @@ -23,6 +23,7 @@ import whelk.search.ElasticFind import whelk.util.PropertyLoader import whelk.util.Romanizer +import java.time.Instant import java.time.ZoneId /** @@ -251,7 +252,7 @@ class Whelk { return doc } - Map bulkLoad(Collection ids) { + Map bulkLoad(Collection ids, Instant asOf = null) { def idMap = [:] def otherIris = [] List systemIds = [] @@ -276,7 +277,7 @@ class Whelk { idMap.putAll(idToIri) } - return storage.bulkLoad(systemIds) + return storage.bulkLoad(systemIds, asOf) .findAll { id, doc -> !doc.deleted } .collectEntries { id, doc -> [(idMap.getOrDefault(id, id)) : doc]} } diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 1458ebc9cb..7b30e5dbf7 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -131,6 +131,37 @@ class PostgreSQLComponent { FROM unnest(?) AS in_id, lddb l WHERE in_id = l.id """.stripIndent() + + private static final String BULK_LOAD_DOCUMENTS_AS_OF = """ + SELECT + id, data, created, modified, deleted + FROM + lddb__versions v, unnest(?) AS in_id + WHERE + in_id = v.id + AND v.pk = + ( + SELECT + pk + FROM + lddb__versions + WHERE + id = v.id + AND GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) <= ? + ORDER BY GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) DESC + limit 1 + ) + """.stripIndent() + + private static final String GET_DOCUMENT_AS_OF = """ + SELECT id, data, created, modified, deleted + FROM lddb__versions + WHERE id = ? + AND + GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) <= ? + ORDER BY GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) DESC + limit 1 + """.stripIndent() private static final String GET_EMBELLISHED_DOCUMENT = "SELECT data from lddb__embellished where id = ?" @@ -158,7 +189,7 @@ class PostgreSQLComponent { """.stripIndent() private static final String GET_ALL_DOCUMENT_VERSIONS = """ - SELECT id, data, deleted, created, modified, changedBy, changedIn, pk + SELECT id, data, deleted, created, modified, changedBy, changedIn, pk, GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) as modTime FROM lddb__versions WHERE id = ? ORDER BY GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) ASC @@ -1974,6 +2005,10 @@ class PostgreSQLComponent { return load(id, null) } + Document loadAsOf(String id, Timestamp asOf) { + return loadFromSql(GET_DOCUMENT_AS_OF, [1: id, 2: asOf]) + } + Document load(String id, String version) { Document doc if (version && version.isInteger()) { @@ -1993,21 +2028,40 @@ class PostgreSQLComponent { return doc } - Map bulkLoad(Iterable systemIds) { + Map bulkLoad(Iterable systemIds, Instant asOf = null) { return withDbConnection { Connection connection = getMyConnection() PreparedStatement preparedStatement = null ResultSet rs = null try { - preparedStatement = connection.prepareStatement(BULK_LOAD_DOCUMENTS) - preparedStatement.setArray(1, connection.createArrayOf("TEXT", systemIds as String[])) - rs = preparedStatement.executeQuery() - SortedMap result = new TreeMap<>() - while(rs.next()) { - result[rs.getString("id")] = assembleDocument(rs) + // The latest version of every document + if(asOf == null) { + preparedStatement = connection.prepareStatement(BULK_LOAD_DOCUMENTS) + preparedStatement.setArray(1, connection.createArrayOf("TEXT", systemIds as String[])) + + rs = preparedStatement.executeQuery() + SortedMap result = new TreeMap<>() + while (rs.next()) { + result[rs.getString("id")] = assembleDocument(rs) + } + return result + } else { // Every document as it looked at time 'asOf' + preparedStatement = connection.prepareStatement(BULK_LOAD_DOCUMENTS_AS_OF) + preparedStatement.setArray(1, connection.createArrayOf("TEXT", systemIds as String[])) + preparedStatement.setTimestamp(2, Timestamp.from(asOf)) + + System.err.println(preparedStatement) + + rs = preparedStatement.executeQuery() + SortedMap result = new TreeMap<>() + while (rs.next()) { + result[rs.getString("id")] = assembleDocument(rs) + } + System.err.println(" ** GAVE: " + result) + return result } - return result + } finally { close(rs, preparedStatement) } @@ -2433,6 +2487,9 @@ class PostgreSQLComponent { if (items.value instanceof Long) { selectstmt.setLong((Integer) items.key, (Long) items.value) } + if (items.value instanceof Timestamp) { + selectstmt.setTimestamp((Integer) items.key, (Timestamp) items.value) + } } log.trace("Executing query") rs = selectstmt.executeQuery() @@ -2498,7 +2555,7 @@ class PostgreSQLComponent { while (rs.next()) { def doc = assembleDocument(rs) doc.version = v++ - docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"), rs.getInt("pk"))) + docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"), rs.getInt("pk"), rs.getTimestamp("modTime"))) } } finally { close(rs, selectstmt) diff --git a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java index c9665b030e..98e2455109 100644 --- a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java +++ b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java @@ -2,6 +2,8 @@ import whelk.Document; +import java.sql.Timestamp; + /** * Represents a version of a record, including the out-of-record info (like the changedBy column) */ @@ -10,10 +12,13 @@ public class DocumentVersion { public String changedBy; public String changedIn; public int versionID; - public DocumentVersion(Document doc, String changedBy, String changedIn, int versionID) { + + public Timestamp versionWriteTime; + public DocumentVersion(Document doc, String changedBy, String changedIn, int versionID, Timestamp versionWriteTime) { this.doc = doc; this.changedBy = changedBy; this.changedIn = changedIn; this.versionID = versionID; + this.versionWriteTime = versionWriteTime; } } From 0bcf5567a470d7077c30ac417bdddfbb6c55594a Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 8 May 2023 14:51:37 +0200 Subject: [PATCH 13/58] First real working CXZ rule (sort of). --- .../housekeeping/NotificationGenerator.groovy | 89 ++++++++++++------- .../component/PostgreSQLComponent.groovy | 3 - 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 9ef3e583a2..97421331c6 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -1,6 +1,7 @@ package whelk.housekeeping import whelk.Document +import whelk.JsonLd import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log @@ -159,9 +160,11 @@ class NotificationGenerator extends HouseKeeper { ] }*/ - List triggered = changeMatchesAnyTrigger(id, fromVersion, untilVersion, user, library) + List triggered = changeMatchesAnyTrigger( + id, fromVersion.versionWriteTime.toInstant(), + untilVersion.versionWriteTime.toInstant(), user, library) if (triggered) { - whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), ["triggered" : triggered]) + whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), ["triggered" : triggered]) // TODO: Use until as created time!! System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) } } @@ -170,16 +173,16 @@ class NotificationGenerator extends HouseKeeper { } /** - * '*version' parameters are the two relevant versions (before and after). + * 'from' and 'until' are the instants of writing for the changed record (the previous and current versions) * 'user' is the user-data map for a user (which includes their selection of triggers). * 'library' is a library ("sigel") holding the instance in question. * - * This function answers the question: Has 'user' requested to be notified of the change between - * 'fromVersion' and 'untilVersion' for instances held by 'library'? + * This function answers the question: Has 'user' requested to be notified of the occurred changes between + * 'from' and 'until' for instances held by 'library'? * * Returns the URIs of all triggered rules/triggers. */ - private List changeMatchesAnyTrigger(String instanceId, DocumentVersion fromVersion, DocumentVersion untilVersion, Map user, String library) { + private List changeMatchesAnyTrigger(String instanceId, Instant from, Instant until, Map user, String library) { List triggeredTriggers = [] @@ -203,7 +206,7 @@ class NotificationGenerator extends HouseKeeper { if (! triggerObject instanceof String) return String triggerUri = (String) triggerObject - if (triggerIsTriggered(instanceId, fromVersion, untilVersion, triggerUri)) + if (triggerIsTriggered(instanceId, from, until, triggerUri)) triggeredTriggers.add(triggerUri) } } @@ -212,29 +215,42 @@ class NotificationGenerator extends HouseKeeper { } /** - * Do the changes between 'fromVersion' and 'untilVersion' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? + * Do changes to the graph between times 'from' and 'until' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? */ - private boolean triggerIsTriggered(String instanceId, DocumentVersion fromVersion, DocumentVersion untilVersion, String triggerUri) { + private boolean triggerIsTriggered(String instanceId, Instant from, Instant until, String triggerUri) { + + // Load the two versions (old/new) of the instance + Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(from)) + // If a depender is created after a dependency, it will ofc not have existed at the original writing time + // of the dependency, if so, simply load the first available version of the depender. + if (instanceBeforeChange == null) + instanceBeforeChange = whelk.getStorage().load(instanceId, "0") + Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(until)) + switch (triggerUri) { - case "https://id.kb.se/notificationtriggers/primarycontribution": - //whelk.getStorage().load - //whelk.embellish() - //fromVersion. - //Document from = fromVersion.doc.clone() - //Document until = untilVersion.doc.clone() - Document affectedInstance = whelk.getStorage().loadAsOf(instanceId, fromVersion.versionWriteTime) + case "https://id.kb.se/notificationtriggers/primarycontribution": { + // Embellish both the new and old versions with historic dependencies from their respective times + historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], from, 2) + historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], until, 2) - // If a depender is created after a dependency, it will ofc not have existed at the original writing time - // of the dependency, if so, simply load the first available version of the depender. - if (affectedInstance == null) - affectedInstance = whelk.getStorage().load(instanceId, "0") + Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) + Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) - //System.err.println("**** Loaded historic version of INSTANCE: " + affectedInstance.getDataAsString()) + if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || !contributionsAfter instanceof List) + return false - historicEmbellish(affectedInstance, ["instanceOf", "contribution", "agent"], fromVersion.versionWriteTime.toInstant()) - return true + for (Object contrBefore : contributionsBefore) { + for (Object contrAfter : contributionsAfter) { + if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution") ) { + if (!contrBefore.equals(contrAfter)) + return true + } + } + } break + } + } return false } @@ -244,19 +260,27 @@ class NotificationGenerator extends HouseKeeper { * The full general embellish code can not help us here, because it is based on the idea of cached cards, * which can (and must!) only cache the latest/current data for each card, which isn't what we need here * (we need to embellish older historic data). + * + * 'passes' means the number of times to collect URIs and expand the document. In practice it is the limit + * of "number of links" away we can look. + * + * This function mutates docToEmbellish */ - private historicEmbellish(Document doc, List properties, Instant asOf) { - List graphList = doc.data["@graph"] - - System.err.println("**** WILL NOW SCAN FOR LINKS:\n\t" + doc.getDataAsString()) + private historicEmbellish(Document docToEmbellish, List properties, Instant asOf, int passes) { + List graphListToEmbellish = docToEmbellish.data["@graph"] - Set uris = findLinkedURIs(graphList, properties) + for (int i = 0; i < passes; ++i) { + Set uris = findLinkedURIs(graphListToEmbellish, properties) - System.err.println("\tFound links (with chosen properties): " + uris) - - Map linkedDocumentsByUri = whelk.bulkLoad(uris, asOf) + Map linkedDocumentsByUri = whelk.bulkLoad(uris, asOf) + linkedDocumentsByUri.each { + List linkedGraphList = it.value.data["@graph"] + if (linkedGraphList.size() > 1) + graphListToEmbellish.add(linkedGraphList[1]) + } + } - //System.err.println("Was able to fetch historic data for: " + linkedDocumentsByUri.keySet()) + docToEmbellish.data = JsonLd.frame(docToEmbellish.getCompleteId(), docToEmbellish.data) } private Set findLinkedURIs(Object node, List properties) { @@ -278,7 +302,6 @@ class NotificationGenerator extends HouseKeeper { } private List getLinkIfAny(Object node) { - System.err.println("\tCHECK getLinkIfAny with " + node) List uris = [] if (node instanceof Map) { if (node.containsKey("@id")) { diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 7b30e5dbf7..665ebec63c 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -2051,14 +2051,11 @@ class PostgreSQLComponent { preparedStatement.setArray(1, connection.createArrayOf("TEXT", systemIds as String[])) preparedStatement.setTimestamp(2, Timestamp.from(asOf)) - System.err.println(preparedStatement) - rs = preparedStatement.executeQuery() SortedMap result = new TreeMap<>() while (rs.next()) { result[rs.getString("id")] = assembleDocument(rs) } - System.err.println(" ** GAVE: " + result) return result } From bd6aa3475a286a8ff0d6716b72495b322c296278 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 9 May 2023 10:41:39 +0200 Subject: [PATCH 14/58] Cleaning up. --- .../housekeeping/NotificationGenerator.groovy | 23 +++++++++++-------- .../00000021-add-notifications-table.plsql | 10 ++++---- .../component/PostgreSQLComponent.groovy | 16 +++++++------ 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 97421331c6..1743f38fcb 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -79,7 +79,7 @@ class NotificationGenerator extends HouseKeeper { Timestamp until = Timestamp.from(Instant.now()) // Then fetch all changed IDs within that interval - sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified BETWEEN ? AND ? );" + sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -164,7 +164,10 @@ class NotificationGenerator extends HouseKeeper { id, fromVersion.versionWriteTime.toInstant(), untilVersion.versionWriteTime.toInstant(), user, library) if (triggered) { - whelk.getStorage().insertNotification(untilVersion.versionID, user["id"].toString(), ["triggered" : triggered]) // TODO: Use until as created time!! + whelk.getStorage().insertNotification( + untilVersion.versionID, fromVersion.versionID, + user["id"].toString(), ["triggered" : triggered], + untilVersion.versionWriteTime) System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) } } @@ -231,8 +234,8 @@ class NotificationGenerator extends HouseKeeper { case "https://id.kb.se/notificationtriggers/primarycontribution": { // Embellish both the new and old versions with historic dependencies from their respective times - historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], from, 2) - historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], until, 2) + historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], from) + historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], until) Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) @@ -261,16 +264,17 @@ class NotificationGenerator extends HouseKeeper { * which can (and must!) only cache the latest/current data for each card, which isn't what we need here * (we need to embellish older historic data). * - * 'passes' means the number of times to collect URIs and expand the document. In practice it is the limit - * of "number of links" away we can look. - * * This function mutates docToEmbellish */ - private historicEmbellish(Document docToEmbellish, List properties, Instant asOf, int passes) { + private historicEmbellish(Document docToEmbellish, List properties, Instant asOf) { List graphListToEmbellish = docToEmbellish.data["@graph"] + Set alreadyLoadedURIs = [] - for (int i = 0; i < passes; ++i) { + for (int i = 0; i < properties.size(); ++i) { Set uris = findLinkedURIs(graphListToEmbellish, properties) + uris.removeAll(alreadyLoadedURIs) + if (uris.isEmpty()) + break Map linkedDocumentsByUri = whelk.bulkLoad(uris, asOf) linkedDocumentsByUri.each { @@ -278,6 +282,7 @@ class NotificationGenerator extends HouseKeeper { if (linkedGraphList.size() > 1) graphListToEmbellish.add(linkedGraphList[1]) } + alreadyLoadedURIs.addAll(uris) } docToEmbellish.data = JsonLd.frame(docToEmbellish.getCompleteId(), docToEmbellish.data) diff --git a/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql index 332f69682d..282877cda6 100644 --- a/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql +++ b/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql @@ -26,13 +26,15 @@ UPDATE lddb__schema SET version = new_version; ALTER TABLE lddb__versions ADD UNIQUE (pk); CREATE TABLE IF NOT EXISTS lddb__notifications ( pk SERIAL PRIMARY KEY, - versionid INTEGER, - userid TEXT, - changes jsonb not null, - handled BOOLEAN DEFAULT FALSE, + versionid INTEGER NOT NULL, + baseversionid INTEGER NOT NULL, + userid TEXT NOT NULL, + triggers jsonb NOT NULL, + handled BOOLEAN DEFAULT FALSE NOT NULL, created timestamp with time zone DEFAULT now() NOT NULL, CONSTRAINT version_fk FOREIGN KEY (versionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, + CONSTRAINT baseversion_fk FOREIGN KEY (baseversionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, CONSTRAINT user_fk FOREIGN KEY (userid) REFERENCES lddb__user_data(id) ON DELETE CASCADE ); CREATE INDEX idx_notifications_user ON lddb__notifications USING BTREE (userid); diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 665ebec63c..c9dde85263 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -437,12 +437,12 @@ class PostgreSQLComponent { """.stripIndent() private static final String INSERT_NOTIFICATION = """ - INSERT INTO lddb__notifications (versionid, userid, changes) - VALUES (?, ?, ?) + INSERT INTO lddb__notifications (versionid, baseversionid, userid, triggers, created) + VALUES (?, ?, ?, ?, ?) """.stripIndent() private static final String GET_NOTICFICATIONS_FOR_USER = """ - SELECT n.pk, n.changes, n.handled, v.data#>>'{@graph,1,@id}' thingid + SELECT n.pk, n.triggers, n.handled, v.data#>>'{@graph,1,@id}' thingid FROM lddb__notifications n LEFT JOIN lddb__versions v ON n.versionid = v.pk @@ -1472,7 +1472,7 @@ class PostgreSQLComponent { Map notification = [ "notificationID": rs.getInt("pk"), "mainEntityID" : rs.getString("thingid"), - "reason" : mapper.readValue(rs.getString("changes"), Map), + "triggers" : mapper.readValue(rs.getString("triggers"), Map), "handled" : rs.getBoolean("handled") ] results.add(notification) @@ -1484,15 +1484,17 @@ class PostgreSQLComponent { } } - boolean insertNotification(int versionID, String userID, Map changes) { + boolean insertNotification(int versionID, int baseVersionID, String userID, Map triggers, Timestamp time) { return withDbConnection { Connection connection = getMyConnection() PreparedStatement preparedStatement = null try { preparedStatement = connection.prepareStatement(INSERT_NOTIFICATION) preparedStatement.setInt(1, versionID) - preparedStatement.setString(2, userID) - preparedStatement.setObject(3, mapper.writeValueAsString(changes), OTHER) + preparedStatement.setInt(2, baseVersionID) + preparedStatement.setString(3, userID) + preparedStatement.setObject(4, mapper.writeValueAsString(triggers), OTHER) + preparedStatement.setTimestamp(5, time) return (preparedStatement.executeUpdate() == 1) } finally { close(preparedStatement) From 6c1dbc57b88aa7aa8e9fa5a0fce5c6c228da978a Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 9 May 2023 13:26:58 +0200 Subject: [PATCH 15/58] Fix some inteval problems and add the worktitle rule --- .../housekeeping/NotificationGenerator.groovy | 51 +++++++++++++++---- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 1743f38fcb..28e6b550a7 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -86,9 +86,12 @@ class NotificationGenerator extends HouseKeeper { statement.setTimestamp(2, until) statement.setFetchSize(512) resultSet = statement.executeQuery() + // If both an instance and one of it's dependencies are affected within the same interval, we will + // (without this check) try to generate notifications for said instance twice. + Set affectedInstanceIDs = [] while (resultSet.next()) { String id = resultSet.getString("id") - generateNotificationsForChangedID(id, libraryToUsers, from.toInstant()) + generateNotificationsForChangedID(id, libraryToUsers, from.toInstant(), until.toInstant(), affectedInstanceIDs) } // Finally, clean out any notifications that are too old @@ -105,7 +108,7 @@ class NotificationGenerator extends HouseKeeper { } } - private void generateNotificationsForChangedID(String id, Map libraryToUsers, Instant since) { + private void generateNotificationsForChangedID(String id, Map libraryToUsers, Instant since, Instant until, Set affectedInstanceIDs) { // "versions" come sorted by ascending modification time, so oldest version first. // We want to pick the "from version" (the base for which this notice details changes) // as the last saved version *before* the sought interval. @@ -128,7 +131,11 @@ class NotificationGenerator extends HouseKeeper { String dependerID = it[0] String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { - generateNotificationsForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion) + // If we've not already created a notification for this instance! + if (!affectedInstanceIDs.contains(dependerID)) { + affectedInstanceIDs.add(dependerID) + generateNotificationsForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion, until) + } } } @@ -138,7 +145,8 @@ class NotificationGenerator extends HouseKeeper { * Generate notice for a bibliographic instance. Beware: fromVersion and untilVersion may not be * _of this document_ (id), but rather of a document this instance depends on! */ - private void generateNotificationsForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, DocumentVersion untilVersion) { + private void generateNotificationsForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, + DocumentVersion untilVersion, Instant creationTime) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { List users = (List) libraryToUsers[library] @@ -162,13 +170,13 @@ class NotificationGenerator extends HouseKeeper { List triggered = changeMatchesAnyTrigger( id, fromVersion.versionWriteTime.toInstant(), - untilVersion.versionWriteTime.toInstant(), user, library) + creationTime, user, library) if (triggered) { whelk.getStorage().insertNotification( untilVersion.versionID, fromVersion.versionID, user["id"].toString(), ["triggered" : triggered], - untilVersion.versionWriteTime) - System.err.println("STORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID) + Timestamp.from(creationTime)) + System.err.println("\tSTORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID + " created: " + creationTime) } } } @@ -233,27 +241,48 @@ class NotificationGenerator extends HouseKeeper { switch (triggerUri) { case "https://id.kb.se/notificationtriggers/primarycontribution": { - // Embellish both the new and old versions with historic dependencies from their respective times historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], from) historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], until) Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) - if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || !contributionsAfter instanceof List) + if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || ! contributionsAfter instanceof List) return false for (Object contrBefore : contributionsBefore) { for (Object contrAfter : contributionsAfter) { if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution") ) { - if (!contrBefore.equals(contrAfter)) - return true + if ( contributionsBefore["agent"] != null && contributionsAfter["agent"] != null) { + if ( + contributionsBefore["agent"]["familyName"] != contributionsAfter["agent"]["familyName"] || + contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || + contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] + ) + return true + } } } } break } + case "https://id.kb.se/notificationtriggers/worktitle": { + historicEmbellish(instanceBeforeChange, ["instanceOf", "hasTitle"], from) + historicEmbellish(instanceAfterChange, ["instanceOf", "hasTitle"], until) + + Object titlesBefore = Document._get(["mainEntity", "instanceOf", "hasTitle"], instanceBeforeChange.data) + Object titlesAfter = Document._get(["mainEntity", "instanceOf", "hasTitle"], instanceAfterChange.data) + + if (titlesBefore == null || titlesAfter == null || ! titlesBefore instanceof List || ! titlesAfter instanceof List) + return false + + if (titlesAfter as Set != titlesBefore as Set) + return true + + break + } + } return false } From bfdc3bcf4cadc851c3ee82ffd205c5213b82e8e7 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 09:58:04 +0200 Subject: [PATCH 16/58] Fix constructors in unit tests. --- .../groovy/whelk/importer/MergeSpec.groovy | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy index c57faeed5d..9de2c4d070 100644 --- a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy +++ b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy @@ -6,6 +6,9 @@ import whelk.JsonLd import whelk.history.DocumentVersion import whelk.history.History +import java.sql.Timestamp +import java.time.Instant + class MergeSpec extends Specification { static final Map CONTEXT_DATA = [ @@ -55,7 +58,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -101,7 +104,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -147,7 +150,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -185,7 +188,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -224,7 +227,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -262,7 +265,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -303,7 +306,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -348,7 +351,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -395,7 +398,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -441,7 +444,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -479,7 +482,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -533,7 +536,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -614,7 +617,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -695,7 +698,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -776,7 +779,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -882,7 +885,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -986,7 +989,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) From 918615296116b8c7ff2e1f1c6b065c6dcbec58d9 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 10:09:40 +0200 Subject: [PATCH 17/58] Fix additional tests. --- .../src/test/groovy/whelk/history/HistorySpec.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy index 455b28a1bc..272ffc409d 100644 --- a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy +++ b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy @@ -7,6 +7,9 @@ import whelk.JsonLd import whelk.util.Jackson import whelk.util.JsonLdSpec +import java.sql.Timestamp +import java.time.Instant + class HistorySpec extends Specification { def "array(set) order does not matter"() { given: @@ -688,7 +691,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -723,7 +726,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -758,7 +761,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -776,7 +779,7 @@ class HistorySpec extends Specification { def ld = new JsonLd(JsonLdSpec.CONTEXT_DATA, display, JsonLdSpec.VOCAB_DATA) def v = versions.collect { data -> - new DocumentVersion(new Document(data), '', '', 0) + new DocumentVersion(new Document(data), '', '', 0, Timestamp.from(Instant.EPOCH)) } def history = new History(v, ld) From da91b2d63991ca26324a191b4059b54b49ca0a53 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 10:38:10 +0200 Subject: [PATCH 18/58] Fix even more tests. --- rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy index 03e0a29b52..9114bc55ed 100644 --- a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy +++ b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy @@ -20,6 +20,8 @@ import javax.servlet.ServletOutputStream import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse import javax.servlet.http.HttpServletResponseWrapper +import java.sql.Timestamp +import java.time.Instant import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND @@ -817,8 +819,8 @@ class CrudSpec extends Specification { } storage.loadDocumentHistory(_) >> { [ - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", "", 0), - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", "", 0), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", "", 0, Timestamp.from(Instant.EPOCH)), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", "", 0, Timestamp.from(Instant.EPOCH)), ] } From 6efcf59154cf838852cfaef68e1c8db83adc1d9d Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 10:46:30 +0200 Subject: [PATCH 19/58] Naming things. --- .../test/groovy/whelk/importer/MergeSpec.groovy | 4 ++-- .../housekeeping/NotificationGenerator.groovy | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy index aae834f05d..9cba05a03b 100644 --- a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy +++ b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy @@ -1081,7 +1081,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1155,7 +1155,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 28e6b550a7..96039115de 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -228,21 +228,21 @@ class NotificationGenerator extends HouseKeeper { /** * Do changes to the graph between times 'from' and 'until' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? */ - private boolean triggerIsTriggered(String instanceId, Instant from, Instant until, String triggerUri) { + private boolean triggerIsTriggered(String instanceId, Instant before, Instant after, String triggerUri) { // Load the two versions (old/new) of the instance - Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(from)) + Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) // If a depender is created after a dependency, it will ofc not have existed at the original writing time // of the dependency, if so, simply load the first available version of the depender. if (instanceBeforeChange == null) instanceBeforeChange = whelk.getStorage().load(instanceId, "0") - Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(until)) + Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(after)) switch (triggerUri) { case "https://id.kb.se/notificationtriggers/primarycontribution": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], from) - historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], until) + historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], before) + historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], after) Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) @@ -268,8 +268,8 @@ class NotificationGenerator extends HouseKeeper { } case "https://id.kb.se/notificationtriggers/worktitle": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "hasTitle"], from) - historicEmbellish(instanceAfterChange, ["instanceOf", "hasTitle"], until) + historicEmbellish(instanceBeforeChange, ["instanceOf", "hasTitle"], before) + historicEmbellish(instanceAfterChange, ["instanceOf", "hasTitle"], after) Object titlesBefore = Document._get(["mainEntity", "instanceOf", "hasTitle"], instanceBeforeChange.data) Object titlesAfter = Document._get(["mainEntity", "instanceOf", "hasTitle"], instanceAfterChange.data) From fee942f6fdd0d8beb48744fc4c590be1c048659e Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 13:05:50 +0200 Subject: [PATCH 20/58] Add intendedAudience notification trigger. --- .../housekeeping/NotificationGenerator.groovy | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 96039115de..3b1068f4ee 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -283,6 +283,22 @@ class NotificationGenerator extends HouseKeeper { break } + case "https://id.kb.se/notificationtriggers/intendedaudience": { + historicEmbellish(instanceBeforeChange, ["instanceOf", "intendedAudience"], before) + historicEmbellish(instanceAfterChange, ["instanceOf", "intendedAudience"], after) + + Object audienceBefore = Document._get(["mainEntity", "instanceOf", "intendedAudience"], instanceBeforeChange.data) + Object audienceAfter = Document._get(["mainEntity", "instanceOf", "intendedAudience"], instanceAfterChange.data) + + if (audienceBefore == null || audienceAfter == null || ! audienceBefore instanceof List || ! audienceAfter instanceof List) + return false + + if (audienceAfter as Set != audienceBefore as Set) + return true + + break + } + } return false } From 1f959b81b69c54a275e84498e4e8da01b2e95d75 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 13:37:21 +0200 Subject: [PATCH 21/58] Add DDK notification trigger --- .../housekeeping/NotificationGenerator.groovy | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 3b1068f4ee..02ddb73558 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -299,6 +299,25 @@ class NotificationGenerator extends HouseKeeper { break } + case "https://id.kb.se/notificationtriggers/ddc": { + historicEmbellish(instanceBeforeChange, ["instanceOf", "classification"], before) + historicEmbellish(instanceAfterChange, ["instanceOf", "classification"], after) + + Object classificationBefore = Document._get(["mainEntity", "instanceOf", "classification"], instanceBeforeChange.data) + Object classificationAfter = Document._get(["mainEntity", "instanceOf", "classification"], instanceAfterChange.data) + + if (classificationBefore == null || classificationAfter == null || ! classificationBefore instanceof List || ! classificationAfter instanceof List) + return false + + if ( + classificationAfter.findAll( it -> it["@type"] == "ClassificationDdc" ) as Set != + classificationBefore.findAll( it -> it["@type"] == "ClassificationDdc" ) as Set + ) + return true + + break + } + } return false } From 096843308f1d129b78e060bd0d9961da17f0714f Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 10 May 2023 14:07:49 +0200 Subject: [PATCH 22/58] More notification rules. --- .../housekeeping/NotificationGenerator.groovy | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 02ddb73558..827d891356 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -318,6 +318,47 @@ class NotificationGenerator extends HouseKeeper { break } + case "https://id.kb.se/notificationtriggers/sab": { + historicEmbellish(instanceBeforeChange, ["instanceOf", "classification"], before) + historicEmbellish(instanceAfterChange, ["instanceOf", "classification"], after) + + Object classificationBefore = Document._get(["mainEntity", "instanceOf", "classification"], instanceBeforeChange.data) + Object classificationAfter = Document._get(["mainEntity", "instanceOf", "classification"], instanceAfterChange.data) + + if (classificationBefore == null || classificationAfter == null || ! classificationBefore instanceof List || ! classificationAfter instanceof List) + return false + + Collection sabBefore = classificationAfter.findAll( it -> it["inScheme"] == "https://id.kb.se/term/kssb" ) + Collection sabAfter = classificationAfter.findAll( it -> it["inScheme"] == "https://id.kb.se/term/kssb" ) + sabBefore = sabBefore.findAll { + if (it["code"] == null) + return false + return ((String) it["code"]).matches(".+/[A-Z].*") + } + sabAfter = sabAfter.findAll { + if (it["code"] == null) + return false + return ((String) it["code"]).matches(".+/[A-Z].*") + } + return !sabAfter.containsAll(sabBefore) + } + + case "https://id.kb.se/notificationtriggers/instancetitle": { + historicEmbellish(instanceBeforeChange, ["hasTitle"], before) + historicEmbellish(instanceAfterChange, ["hasTitle"], after) + + Object titlesBefore = Document._get(["mainEntity", "hasTitle"], instanceBeforeChange.data) + Object titlesAfter = Document._get(["mainEntity", "hasTitle"], instanceAfterChange.data) + + if (titlesBefore == null || titlesAfter == null || ! titlesBefore instanceof List || ! titlesAfter instanceof List) + return false + + if (titlesAfter as Set != titlesBefore as Set) + return true + + break + } + } return false } From 3901050b20f3fda3031288e19000d96752569c9b Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 21 Sep 2023 12:50:45 +0200 Subject: [PATCH 23/58] Begin the process of repurposing the CXZ generation code into 'CXZ sending of already generated markers', which is remarkably similar. --- .../groovy/whelk/importer/MergeSpec.groovy | 38 ++--- ...rator.groovy => NotificationSender.groovy} | 137 +++--------------- .../whelk/housekeeping/WebInterface.groovy | 2 +- .../00000021-add-notifications-table.plsql | 45 ------ .../whelk/rest/api/NotificationsAPI.groovy | 66 --------- rest/src/main/webapp/WEB-INF/web.xml | 14 -- .../component/PostgreSQLComponent.groovy | 80 +--------- .../groovy/whelk/history/DocumentVersion.java | 4 +- .../groovy/whelk/history/HistorySpec.groovy | 8 +- 9 files changed, 44 insertions(+), 350 deletions(-) rename housekeeping/src/main/groovy/whelk/housekeeping/{NotificationGenerator.groovy => NotificationSender.groovy} (64%) delete mode 100644 librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql delete mode 100644 rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy diff --git a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy index ee8b54ad08..1bf0be23c5 100644 --- a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy +++ b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy @@ -58,7 +58,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -104,7 +104,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -150,7 +150,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -188,7 +188,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -227,7 +227,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -265,7 +265,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -306,7 +306,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -351,7 +351,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -398,7 +398,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -444,7 +444,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -482,7 +482,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -536,7 +536,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -617,7 +617,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -698,7 +698,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -779,7 +779,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -885,7 +885,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -989,7 +989,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1081,7 +1081,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1155,7 +1155,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy similarity index 64% rename from housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy rename to housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 827d891356..fa078f807f 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -16,18 +16,17 @@ import java.time.temporal.ChronoUnit @CompileStatic @Log -class NotificationGenerator extends HouseKeeper { +class NotificationSender extends HouseKeeper { - private static final int DAYS_TO_KEEP_NOTIFICATIONS = 10 private String status = "OK" private Whelk whelk - public NotificationGenerator(Whelk whelk) { + public NotificationSender(Whelk whelk) { this.whelk = whelk } public String getName() { - return "Notifications generator" + return "Notifications sender" } public String getStatusDescription() { @@ -67,7 +66,8 @@ class NotificationGenerator extends HouseKeeper { // This interval, should generally be: From the last generated notice until now. // However, if there are no previously generated notices (near enough in time), use // now - [some pre set value], to avoid scanning the whole catalog. - String sql = "SELECT MAX(created) FROM lddb__notifications;" + + /*String sql = "SELECT MAX(created) FROM lddb__notifications;" statement = connection.prepareStatement(sql) resultSet = statement.executeQuery() Timestamp from = Timestamp.from(Instant.now().minus(DAYS_TO_KEEP_NOTIFICATIONS, ChronoUnit.DAYS)) @@ -75,11 +75,12 @@ class NotificationGenerator extends HouseKeeper { Timestamp lastCreated = resultSet.getTimestamp(1) if (lastCreated && lastCreated.after(from)) from = lastCreated - } + }*/ + Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) Timestamp until = Timestamp.from(Instant.now()) // Then fetch all changed IDs within that interval - sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" + String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -91,15 +92,8 @@ class NotificationGenerator extends HouseKeeper { Set affectedInstanceIDs = [] while (resultSet.next()) { String id = resultSet.getString("id") - generateNotificationsForChangedID(id, libraryToUsers, from.toInstant(), until.toInstant(), affectedInstanceIDs) + sendNotificationsForChangedID(id, libraryToUsers, from.toInstant(), until.toInstant(), affectedInstanceIDs) } - - // Finally, clean out any notifications that are too old - sql = "DELETE FROM lddb__notifications WHERE created < ?" - statement = connection.prepareStatement(sql) - statement.setTimestamp(1, Timestamp.from(Instant.now().minus(DAYS_TO_KEEP_NOTIFICATIONS, ChronoUnit.DAYS))) - statement.executeUpdate() - } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() throw e @@ -108,7 +102,7 @@ class NotificationGenerator extends HouseKeeper { } } - private void generateNotificationsForChangedID(String id, Map libraryToUsers, Instant since, Instant until, Set affectedInstanceIDs) { + private void sendNotificationsForChangedID(String id, Map libraryToUsers, Instant since, Instant until, Set affectedInstanceIDs) { // "versions" come sorted by ascending modification time, so oldest version first. // We want to pick the "from version" (the base for which this notice details changes) // as the last saved version *before* the sought interval. @@ -131,10 +125,10 @@ class NotificationGenerator extends HouseKeeper { String dependerID = it[0] String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { - // If we've not already created a notification for this instance! + // If we've not already sent a notification for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - generateNotificationsForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion, until) + sendNotificationsForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion, until) } } } @@ -142,10 +136,10 @@ class NotificationGenerator extends HouseKeeper { } /** - * Generate notice for a bibliographic instance. Beware: fromVersion and untilVersion may not be + * Send notices for a bibliographic instance. Beware: fromVersion and untilVersion may not be * _of this document_ (id), but rather of a document this instance depends on! */ - private void generateNotificationsForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, + private void sendNotificationsForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, DocumentVersion untilVersion, Instant creationTime) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { @@ -172,11 +166,7 @@ class NotificationGenerator extends HouseKeeper { id, fromVersion.versionWriteTime.toInstant(), creationTime, user, library) if (triggered) { - whelk.getStorage().insertNotification( - untilVersion.versionID, fromVersion.versionID, - user["id"].toString(), ["triggered" : triggered], - Timestamp.from(creationTime)) - System.err.println("\tSTORED NOTICE FOR USER " + user["id"].toString() + " version: " + untilVersion.versionID + " created: " + creationTime) + System.err.println("\tSEND NOTICE FOR USER " + user["id"].toString()) } } } @@ -244,7 +234,8 @@ class NotificationGenerator extends HouseKeeper { historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], before) historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], after) - Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) + // Check for pre-made change notes instead. + /*Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || ! contributionsAfter instanceof List) @@ -263,99 +254,7 @@ class NotificationGenerator extends HouseKeeper { } } } - } - break - } - - case "https://id.kb.se/notificationtriggers/worktitle": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "hasTitle"], before) - historicEmbellish(instanceAfterChange, ["instanceOf", "hasTitle"], after) - - Object titlesBefore = Document._get(["mainEntity", "instanceOf", "hasTitle"], instanceBeforeChange.data) - Object titlesAfter = Document._get(["mainEntity", "instanceOf", "hasTitle"], instanceAfterChange.data) - - if (titlesBefore == null || titlesAfter == null || ! titlesBefore instanceof List || ! titlesAfter instanceof List) - return false - - if (titlesAfter as Set != titlesBefore as Set) - return true - - break - } - - case "https://id.kb.se/notificationtriggers/intendedaudience": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "intendedAudience"], before) - historicEmbellish(instanceAfterChange, ["instanceOf", "intendedAudience"], after) - - Object audienceBefore = Document._get(["mainEntity", "instanceOf", "intendedAudience"], instanceBeforeChange.data) - Object audienceAfter = Document._get(["mainEntity", "instanceOf", "intendedAudience"], instanceAfterChange.data) - - if (audienceBefore == null || audienceAfter == null || ! audienceBefore instanceof List || ! audienceAfter instanceof List) - return false - - if (audienceAfter as Set != audienceBefore as Set) - return true - - break - } - - case "https://id.kb.se/notificationtriggers/ddc": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "classification"], before) - historicEmbellish(instanceAfterChange, ["instanceOf", "classification"], after) - - Object classificationBefore = Document._get(["mainEntity", "instanceOf", "classification"], instanceBeforeChange.data) - Object classificationAfter = Document._get(["mainEntity", "instanceOf", "classification"], instanceAfterChange.data) - - if (classificationBefore == null || classificationAfter == null || ! classificationBefore instanceof List || ! classificationAfter instanceof List) - return false - - if ( - classificationAfter.findAll( it -> it["@type"] == "ClassificationDdc" ) as Set != - classificationBefore.findAll( it -> it["@type"] == "ClassificationDdc" ) as Set - ) - return true - - break - } - - case "https://id.kb.se/notificationtriggers/sab": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "classification"], before) - historicEmbellish(instanceAfterChange, ["instanceOf", "classification"], after) - - Object classificationBefore = Document._get(["mainEntity", "instanceOf", "classification"], instanceBeforeChange.data) - Object classificationAfter = Document._get(["mainEntity", "instanceOf", "classification"], instanceAfterChange.data) - - if (classificationBefore == null || classificationAfter == null || ! classificationBefore instanceof List || ! classificationAfter instanceof List) - return false - - Collection sabBefore = classificationAfter.findAll( it -> it["inScheme"] == "https://id.kb.se/term/kssb" ) - Collection sabAfter = classificationAfter.findAll( it -> it["inScheme"] == "https://id.kb.se/term/kssb" ) - sabBefore = sabBefore.findAll { - if (it["code"] == null) - return false - return ((String) it["code"]).matches(".+/[A-Z].*") - } - sabAfter = sabAfter.findAll { - if (it["code"] == null) - return false - return ((String) it["code"]).matches(".+/[A-Z].*") - } - return !sabAfter.containsAll(sabBefore) - } - - case "https://id.kb.se/notificationtriggers/instancetitle": { - historicEmbellish(instanceBeforeChange, ["hasTitle"], before) - historicEmbellish(instanceAfterChange, ["hasTitle"], after) - - Object titlesBefore = Document._get(["mainEntity", "hasTitle"], instanceBeforeChange.data) - Object titlesAfter = Document._get(["mainEntity", "hasTitle"], instanceAfterChange.data) - - if (titlesBefore == null || titlesAfter == null || ! titlesBefore instanceof List || ! titlesAfter instanceof List) - return false - - if (titlesAfter as Set != titlesBefore as Set) - return true - + }*/ break } diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 4040173f6b..3d4d50b8da 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -30,7 +30,7 @@ public class WebInterface extends HttpServlet { Whelk whelk = Whelk.createLoadedCoreWhelk() houseKeepers = [] - houseKeepers.add(new NotificationGenerator(whelk)) + houseKeepers.add(new NotificationSender(whelk)) for (HouseKeeper hk : houseKeepers) { timer.scheduleAtFixedRate({ diff --git a/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql deleted file mode 100644 index 282877cda6..0000000000 --- a/librisxl-tools/postgresql/migrations/00000021-add-notifications-table.plsql +++ /dev/null @@ -1,45 +0,0 @@ -BEGIN; - -DO $$DECLARE - -- THESE MUST BE CHANGED WHEN YOU COPY THE SCRIPT! - - -- The version you expect the database to have _before_ the migration - old_version numeric := 20; - -- The version the database should have _after_ the migration - new_version numeric := 21; - - -- hands off - existing_version numeric; - -BEGIN - - -- Check existing version -SELECT version from lddb__schema INTO existing_version; -IF ( existing_version <> old_version) THEN - RAISE EXCEPTION 'ASKED TO MIGRATE FROM INCORRECT EXISTING VERSION!'; -ROLLBACK; -END IF; -UPDATE lddb__schema SET version = new_version; - --- ACTUAL SCHEMA CHANGES HERE: - -ALTER TABLE lddb__versions ADD UNIQUE (pk); -CREATE TABLE IF NOT EXISTS lddb__notifications ( - pk SERIAL PRIMARY KEY, - versionid INTEGER NOT NULL, - baseversionid INTEGER NOT NULL, - userid TEXT NOT NULL, - triggers jsonb NOT NULL, - handled BOOLEAN DEFAULT FALSE NOT NULL, - created timestamp with time zone DEFAULT now() NOT NULL, - - CONSTRAINT version_fk FOREIGN KEY (versionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, - CONSTRAINT baseversion_fk FOREIGN KEY (baseversionid) REFERENCES lddb__versions(pk) ON DELETE CASCADE, - CONSTRAINT user_fk FOREIGN KEY (userid) REFERENCES lddb__user_data(id) ON DELETE CASCADE -); -CREATE INDEX idx_notifications_user ON lddb__notifications USING BTREE (userid); -CREATE INDEX idx_notifications_created ON lddb__notifications USING BTREE (created); - -END$$; - -COMMIT; diff --git a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy b/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy deleted file mode 100644 index 2d7e99453d..0000000000 --- a/rest/src/main/groovy/whelk/rest/api/NotificationsAPI.groovy +++ /dev/null @@ -1,66 +0,0 @@ -package whelk.rest.api - -import groovy.util.logging.Log4j2 as Log -import whelk.Whelk -import whelk.util.WhelkFactory - -import javax.servlet.http.HttpServlet -import javax.servlet.http.HttpServletRequest -import javax.servlet.http.HttpServletResponse - -@Log -class NotificationsAPI extends HttpServlet { - - private Whelk whelk - - @Override - void init() { - log.info("Starting Notifications API") - if (!whelk) { - whelk = WhelkFactory.getSingletonWhelk() - } - } - - @Override - void doGet(HttpServletRequest request, HttpServletResponse response) { - Map userInfo = request.getAttribute("user") - if (!isValidUserWithPermission(request, response, userInfo)) - return - - String id = "${userInfo.id}".digest(UserDataAPI.ID_HASH_FUNCTION) - List data = whelk.getStorage().getNotificationsFor(id) - - HttpTools.sendResponse(response, ["notifications": data], "application/json") - } - - @Override - void doPost(HttpServletRequest request, HttpServletResponse response) { - log.info("Handling POST request for ${request.pathInfo}") - - Map userInfo = request.getAttribute("user") - if (!isValidUserWithPermission(request, response, userInfo)) - return - - String userID = "${userInfo.id}".digest(UserDataAPI.ID_HASH_FUNCTION) - String notificationID = request.getPathInfo().replace("/", "") - whelk.getStorage().flipNotificationHandled(userID, Integer.parseInt(notificationID)) - - response.setStatus(HttpServletResponse.SC_OK) - } - - static boolean isValidUserWithPermission(HttpServletRequest request, HttpServletResponse response, Map userInfo) { - if (!userInfo) { - log.info("User authentication failed") - response.sendError(HttpServletResponse.SC_FORBIDDEN, "User authentication failed") - return false - } - - if (!userInfo.containsKey("id")) { - log.info("User check failed: 'id' missing in user") - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Key 'id' missing in user info") - return false - } - - return true - } -} diff --git a/rest/src/main/webapp/WEB-INF/web.xml b/rest/src/main/webapp/WEB-INF/web.xml index 5abdd7fa2f..345a555382 100644 --- a/rest/src/main/webapp/WEB-INF/web.xml +++ b/rest/src/main/webapp/WEB-INF/web.xml @@ -98,10 +98,6 @@ TransliterationAPI whelk.rest.api.TransliterationAPI - - NotificationsAPI - whelk.rest.api.NotificationsAPI - @@ -110,11 +106,6 @@ 1 - - NotificationsAPI - /_notifications/* - - UserDataAPI /_userdata/* @@ -190,11 +181,6 @@ /_userdata/* - - AuthenticationFilterGet - /_notifications/* - - AuthenticationFilterGet /_remotesearch diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index b9b5317944..ac04872292 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -436,24 +436,6 @@ class PostgreSQLComponent { JOIN lddb ON lddb__identifiers.id = lddb.id WHERE lddb__identifiers.iri = ? """.stripIndent() - private static final String INSERT_NOTIFICATION = """ - INSERT INTO lddb__notifications (versionid, baseversionid, userid, triggers, created) - VALUES (?, ?, ?, ?, ?) - """.stripIndent() - - private static final String GET_NOTICFICATIONS_FOR_USER = """ - SELECT n.pk, n.triggers, n.handled, v.data#>>'{@graph,1,@id}' thingid - FROM lddb__notifications n - LEFT JOIN lddb__versions v - ON n.versionid = v.pk - WHERE userid = ? - ORDER BY n.created ASC - """.stripIndent() - - private static final String FLIP_NOTIFICATION_HANDLED = """ - UPDATE lddb__notifications SET handled = NOT handled WHERE userid = ? AND pk = ? - """.stripIndent() - private static final String GET_ALL_LIBRARIES_HOLDING_ID = """ SELECT l.data#>>'{@graph,1,heldBy,@id}' FROM lddb__dependencies d LEFT JOIN lddb l ON d.id = l.id @@ -1457,66 +1439,6 @@ class PostgreSQLComponent { } } - List getNotificationsFor(String userid) { - return withDbConnection { - Connection connection = getMyConnection() - PreparedStatement preparedStatement = null - ResultSet rs = null - try { - preparedStatement = connection.prepareStatement(GET_NOTICFICATIONS_FOR_USER) - preparedStatement.setString(1, userid) - - rs = preparedStatement.executeQuery() - List results = [] - while(rs.next()) { - Map notification = [ - "notificationID": rs.getInt("pk"), - "mainEntityID" : rs.getString("thingid"), - "triggers" : mapper.readValue(rs.getString("triggers"), Map), - "handled" : rs.getBoolean("handled") - ] - results.add(notification) - } - return results - } finally { - close(rs, preparedStatement) - } - } - } - - boolean insertNotification(int versionID, int baseVersionID, String userID, Map triggers, Timestamp time) { - return withDbConnection { - Connection connection = getMyConnection() - PreparedStatement preparedStatement = null - try { - preparedStatement = connection.prepareStatement(INSERT_NOTIFICATION) - preparedStatement.setInt(1, versionID) - preparedStatement.setInt(2, baseVersionID) - preparedStatement.setString(3, userID) - preparedStatement.setObject(4, mapper.writeValueAsString(triggers), OTHER) - preparedStatement.setTimestamp(5, time) - return (preparedStatement.executeUpdate() == 1) - } finally { - close(preparedStatement) - } - } - } - - boolean flipNotificationHandled(String userID, int notificationID) { - return withDbConnection { - Connection connection = getMyConnection() - PreparedStatement preparedStatement = null - try { - preparedStatement = connection.prepareStatement(FLIP_NOTIFICATION_HANDLED) - preparedStatement.setString(1, userID) - preparedStatement.setInt(2, notificationID) - return (preparedStatement.executeUpdate() == 1) - } finally { - close(preparedStatement) - } - } - } - void recalculateDependencies(Document doc) { withDbConnection { saveDependencies(doc, getMyConnection()) @@ -2555,7 +2477,7 @@ class PostgreSQLComponent { while (rs.next()) { def doc = assembleDocument(rs) doc.version = v++ - docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"), rs.getInt("pk"), rs.getTimestamp("modTime"))) + docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"), rs.getTimestamp("modTime"))) } } finally { close(rs, selectstmt) diff --git a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java index 98e2455109..037f57fffa 100644 --- a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java +++ b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java @@ -11,14 +11,12 @@ public class DocumentVersion { public Document doc; public String changedBy; public String changedIn; - public int versionID; public Timestamp versionWriteTime; - public DocumentVersion(Document doc, String changedBy, String changedIn, int versionID, Timestamp versionWriteTime) { + public DocumentVersion(Document doc, String changedBy, String changedIn, Timestamp versionWriteTime) { this.doc = doc; this.changedBy = changedBy; this.changedIn = changedIn; - this.versionID = versionID; this.versionWriteTime = versionWriteTime; } } diff --git a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy index 272ffc409d..6bac279500 100644 --- a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy +++ b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy @@ -691,7 +691,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -726,7 +726,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -761,7 +761,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) @@ -779,7 +779,7 @@ class HistorySpec extends Specification { def ld = new JsonLd(JsonLdSpec.CONTEXT_DATA, display, JsonLdSpec.VOCAB_DATA) def v = versions.collect { data -> - new DocumentVersion(new Document(data), '', '', 0, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(data), '', '', Timestamp.from(Instant.EPOCH)) } def history = new History(v, ld) From 6778238027b4a47016b688543c4b9dcc6f86deb0 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 21 Sep 2023 13:07:03 +0200 Subject: [PATCH 24/58] Fix unit tests --- .../src/test/groovy/whelk/importer/MergeSpec.groovy | 8 ++++---- rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy index 1bf0be23c5..c99d331328 100644 --- a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy +++ b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy @@ -1218,7 +1218,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1286,7 +1286,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1359,7 +1359,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1422,7 +1422,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) } def history = new History(versions, ld) def incoming = new Document( (Map) diff --git a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy index 9114bc55ed..f04ffd5bcb 100644 --- a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy +++ b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy @@ -819,8 +819,8 @@ class CrudSpec extends Specification { } storage.loadDocumentHistory(_) >> { [ - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", "", 0, Timestamp.from(Instant.EPOCH)), - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", "", 0, Timestamp.from(Instant.EPOCH)), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", "", Timestamp.from(Instant.EPOCH)), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", "", Timestamp.from(Instant.EPOCH)), ] } From dd746f4d623fd5105ef7fb47cdcb133833a13a94 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 29 Sep 2023 11:12:02 +0200 Subject: [PATCH 25/58] Temporary commit --- .../main/groovy/whelk/housekeeping/NotificationSender.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index fa078f807f..14264ec5ba 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -216,7 +216,7 @@ class NotificationSender extends HouseKeeper { } /** - * Do changes to the graph between times 'from' and 'until' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? + * Do changes to the graph between times 'before' and 'after' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? */ private boolean triggerIsTriggered(String instanceId, Instant before, Instant after, String triggerUri) { From e430670189e7177ed8aeeb735e65ee816dec3b85 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 2 Oct 2023 13:26:06 +0200 Subject: [PATCH 26/58] Use 'heldBy' instead of 'library' for notifications requests. --- .../housekeeping/NotificationSender.groovy | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 14264ec5ba..dd53015762 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -36,21 +36,21 @@ class NotificationSender extends HouseKeeper { public void trigger() { // Build a multi-map of library -> list of settings objects for that library's users - Map> libraryToUsers = new HashMap<>(); + Map> heldByToUserSettings = new HashMap<>(); { List allUserSettingStrings = whelk.getStorage().getAllUserData() for (Map settings : allUserSettingStrings) { settings?.requestedNotifications.each { request -> if (!request instanceof Map) return - if (!request["library"]) + if (!request["heldBy"]) return - String library = request["library"] - if (!libraryToUsers.containsKey(library)) - libraryToUsers.put(library, []) - List userSettingsForThisLib = libraryToUsers[library] - userSettingsForThisLib.add(settings) + String heldBy = request["heldBy"] + if (!heldByToUserSettings.containsKey(heldBy)) + heldByToUserSettings.put(heldBy, []) + List userSettingsForThisHeldBy = heldByToUserSettings[heldBy] + userSettingsForThisHeldBy.add(settings) } } } @@ -92,7 +92,7 @@ class NotificationSender extends HouseKeeper { Set affectedInstanceIDs = [] while (resultSet.next()) { String id = resultSet.getString("id") - sendNotificationsForChangedID(id, libraryToUsers, from.toInstant(), until.toInstant(), affectedInstanceIDs) + sendNotificationsForChangedID(id, heldByToUserSettings, from.toInstant(), until.toInstant(), affectedInstanceIDs) } } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() @@ -102,7 +102,7 @@ class NotificationSender extends HouseKeeper { } } - private void sendNotificationsForChangedID(String id, Map libraryToUsers, Instant since, Instant until, Set affectedInstanceIDs) { + private void sendNotificationsForChangedID(String id, Map heldByToUserSettings, Instant since, Instant until, Set affectedInstanceIDs) { // "versions" come sorted by ascending modification time, so oldest version first. // We want to pick the "from version" (the base for which this notice details changes) // as the last saved version *before* the sought interval. @@ -128,7 +128,7 @@ class NotificationSender extends HouseKeeper { // If we've not already sent a notification for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - sendNotificationsForAffectedInstance(dependerID, libraryToUsers, fromVersion, untilVersion, until) + sendNotificationsForAffectedInstance(dependerID, heldByToUserSettings, fromVersion, untilVersion, until) } } } @@ -139,14 +139,13 @@ class NotificationSender extends HouseKeeper { * Send notices for a bibliographic instance. Beware: fromVersion and untilVersion may not be * _of this document_ (id), but rather of a document this instance depends on! */ - private void sendNotificationsForAffectedInstance(String id, Map libraryToUsers, DocumentVersion fromVersion, + private void sendNotificationsForAffectedInstance(String id, Map heldByToUserSettings, DocumentVersion fromVersion, DocumentVersion untilVersion, Instant creationTime) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { - List users = (List) libraryToUsers[library] + List users = (List) heldByToUserSettings[library] if (users) { for (Map user : users) { - /* 'user' is now a map looking something like this: { @@ -155,13 +154,15 @@ class NotificationSender extends HouseKeeper { { "library": "https://libris.kb.se/library/Utb1", "triggers": [ - "https://id.kb.se/notificationtriggers/sab", - "https://id.kb.se/notificationtriggers/primarycontribution" + "https://id.kb.se/changenote/primarytitle" ] } ] }*/ + System.err.println("*** SEND STUFF FOR CHANGED ID: " + id) + System.err.println(" USER SETTINGS ARE : " + user) + List triggered = changeMatchesAnyTrigger( id, fromVersion.versionWriteTime.toInstant(), creationTime, user, library) @@ -230,6 +231,14 @@ class NotificationSender extends HouseKeeper { switch (triggerUri) { + case "https://id.kb.se/changenote/primarytitle": { + historicEmbellish(instanceBeforeChange, ["hasTitle"], before) + historicEmbellish(instanceAfterChange, ["hasTitle"], after) + + + break; + } + case "https://id.kb.se/notificationtriggers/primarycontribution": { historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], before) historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], after) @@ -261,7 +270,7 @@ class NotificationSender extends HouseKeeper { } return false } - + /** * This is a simplified/specialized from of 'embellish', for historic data and using only select properties. * The full general embellish code can not help us here, because it is based on the idea of cached cards, From 34a075cf79c18d87cb41c950db712c06bd6e7404 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 2 Oct 2023 14:43:23 +0200 Subject: [PATCH 27/58] Getting ready to send notifications. --- .../housekeeping/NotificationSender.groovy | 88 ++++++++----------- 1 file changed, 37 insertions(+), 51 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index dd53015762..57abe2ef92 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -12,6 +12,8 @@ import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.Timestamp import java.time.Instant +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit @CompileStatic @@ -160,9 +162,6 @@ class NotificationSender extends HouseKeeper { ] }*/ - System.err.println("*** SEND STUFF FOR CHANGED ID: " + id) - System.err.println(" USER SETTINGS ARE : " + user) - List triggered = changeMatchesAnyTrigger( id, fromVersion.versionWriteTime.toInstant(), creationTime, user, library) @@ -184,7 +183,7 @@ class NotificationSender extends HouseKeeper { * * Returns the URIs of all triggered rules/triggers. */ - private List changeMatchesAnyTrigger(String instanceId, Instant from, Instant until, Map user, String library) { + private List changeMatchesAnyTrigger(String instanceId, Instant from, Instant until, Map user, String heldBy) { List triggeredTriggers = [] @@ -195,10 +194,10 @@ class NotificationSender extends HouseKeeper { if (! request instanceof Map) return - if (! request["library"] instanceof String) + if (! request["heldBy"] instanceof String) return - if (request["library"] != library) + if (request["heldBy"] != heldBy) return if (! request["triggers"] instanceof List) @@ -216,61 +215,44 @@ class NotificationSender extends HouseKeeper { return triggeredTriggers } - /** - * Do changes to the graph between times 'before' and 'after' affect 'instanceId' in such a way as to qualify 'triggerUri' triggered? - */ private boolean triggerIsTriggered(String instanceId, Instant before, Instant after, String triggerUri) { - - // Load the two versions (old/new) of the instance - Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) - // If a depender is created after a dependency, it will ofc not have existed at the original writing time - // of the dependency, if so, simply load the first available version of the depender. - if (instanceBeforeChange == null) - instanceBeforeChange = whelk.getStorage().load(instanceId, "0") Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(after)) + // Populate 'changeNotes' with every note affecting this instance in the specified interval + List changeNotes = new ArrayList<>(); + List notesOnInstance = (List) Document._get(["@graph", 0, "hasChangeNote"], instanceAfterChange.data) + if (notesOnInstance != null) + changeNotes.addAll(notesOnInstance) switch (triggerUri) { - case "https://id.kb.se/changenote/primarytitle": { - historicEmbellish(instanceBeforeChange, ["hasTitle"], before) - historicEmbellish(instanceAfterChange, ["hasTitle"], after) - - - break; + historicEmbellish(instanceAfterChange, ["hasTitle"], after, changeNotes) } + } - case "https://id.kb.se/notificationtriggers/primarycontribution": { - historicEmbellish(instanceBeforeChange, ["instanceOf", "contribution", "agent"], before) - historicEmbellish(instanceAfterChange, ["instanceOf", "contribution", "agent"], after) - - // Check for pre-made change notes instead. - /*Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) - Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) - - if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || ! contributionsAfter instanceof List) - return false - - for (Object contrBefore : contributionsBefore) { - for (Object contrAfter : contributionsAfter) { - if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution") ) { - if ( contributionsBefore["agent"] != null && contributionsAfter["agent"] != null) { - if ( - contributionsBefore["agent"]["familyName"] != contributionsAfter["agent"]["familyName"] || - contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || - contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] - ) - return true - } - } - } - }*/ - break + boolean matches = false + filterChangeNotesNotInInterval(changeNotes, before, after) + changeNotes.each { note -> + note.category.each { category -> + if (category["@id"] == triggerUri) + matches = true } + } + return matches + } + private void filterChangeNotesNotInInterval(List changeNotes, Instant before, Instant after) { + changeNotes.removeAll { changeNote -> + if (changeNote == null) + return true + if (!changeNote.atTime) + return true + Instant atTime = ZonedDateTime.parse( (String) changeNote.atTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant(); + if (atTime.isBefore(before) || atTime.isAfter(after)) + return true + return false } - return false } - + /** * This is a simplified/specialized from of 'embellish', for historic data and using only select properties. * The full general embellish code can not help us here, because it is based on the idea of cached cards, @@ -278,8 +260,9 @@ class NotificationSender extends HouseKeeper { * (we need to embellish older historic data). * * This function mutates docToEmbellish + * This function also collects metadata ChangeNotes from embellished records. */ - private historicEmbellish(Document docToEmbellish, List properties, Instant asOf) { + private historicEmbellish(Document docToEmbellish, List properties, Instant asOf, List changeNotes) { List graphListToEmbellish = docToEmbellish.data["@graph"] Set alreadyLoadedURIs = [] @@ -294,6 +277,9 @@ class NotificationSender extends HouseKeeper { List linkedGraphList = it.value.data["@graph"] if (linkedGraphList.size() > 1) graphListToEmbellish.add(linkedGraphList[1]) + linkedGraphList[0]["hasChangeNote"].each { changeNote -> + changeNotes.add( (Map) changeNote ) + } } alreadyLoadedURIs.addAll(uris) } From a8fff3e3dfc3e317780c67a33f5a3ee04e100d85 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 3 Oct 2023 13:03:08 +0200 Subject: [PATCH 28/58] Add the various categories of changeNotes --- .../housekeeping/NotificationSender.groovy | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 57abe2ef92..d83a06dad5 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -166,7 +166,7 @@ class NotificationSender extends HouseKeeper { id, fromVersion.versionWriteTime.toInstant(), creationTime, user, library) if (triggered) { - System.err.println("\tSEND NOTICE FOR USER " + user["id"].toString()) + System.err.println("\tSEND NOTICE FOR USER " + user["id"].toString() + " : " + triggered) } } } @@ -224,9 +224,27 @@ class NotificationSender extends HouseKeeper { if (notesOnInstance != null) changeNotes.addAll(notesOnInstance) switch (triggerUri) { - case "https://id.kb.se/changenote/primarytitle": { - historicEmbellish(instanceAfterChange, ["hasTitle"], after, changeNotes) - } + case "https://id.kb.se/changenote/primarytitle": + case "https://id.kb.se/changenote/maintitle": + historicEmbellish(instanceAfterChange, ["mainEntity", "hasTitle"], after, changeNotes) + break + case "https://id.kb.se/changenote/primarypublication": + case "https://id.kb.se/changenote/serialtermination": + historicEmbellish(instanceAfterChange, ["publication"], after, changeNotes) + break + case "https://id.kb.se/changenote/intendedaudience": + historicEmbellish(instanceAfterChange, ["mainEntity", "intendedAudience"], after, changeNotes) + break + case "https://id.kb.se/changenote/ddcclassification": + case "https://id.kb.se/changenote/sabclassification": + historicEmbellish(instanceAfterChange, ["mainEntity", "classification"], after, changeNotes) + break + case "https://id.kb.se/changenote/serialrelation": + historicEmbellish(instanceAfterChange, ["mainEntity", "precededBy", "succeededBy"], after, changeNotes) + break + /*case "https://id.kb.se/changenote/primarycontribution": { + historicEmbellish(instanceAfterChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) + }*/ } boolean matches = false From bfd815832d2601c688ec33c3a6784ce2e47ede6e Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 3 Oct 2023 14:21:07 +0200 Subject: [PATCH 29/58] Progress towards email notifications. --- .../housekeeping/NotificationSender.groovy | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index d83a06dad5..f3d0857950 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -42,6 +42,8 @@ class NotificationSender extends HouseKeeper { { List allUserSettingStrings = whelk.getStorage().getAllUserData() for (Map settings : allUserSettingStrings) { + if (!settings["notificationEmail"]) + return settings?.requestedNotifications.each { request -> if (!request instanceof Map) return @@ -57,6 +59,8 @@ class NotificationSender extends HouseKeeper { } } + //System.err.println("Users map:\n\n" + heldByToUserSettings + "\n\n") + Connection connection PreparedStatement statement ResultSet resultSet @@ -92,9 +96,11 @@ class NotificationSender extends HouseKeeper { // If both an instance and one of it's dependencies are affected within the same interval, we will // (without this check) try to generate notifications for said instance twice. Set affectedInstanceIDs = [] + Map> notificationsByUser = new HashMap<>() while (resultSet.next()) { String id = resultSet.getString("id") - sendNotificationsForChangedID(id, heldByToUserSettings, from.toInstant(), until.toInstant(), affectedInstanceIDs) + generateNotificationsForChangedID(id, heldByToUserSettings, from.toInstant(), + until.toInstant(), affectedInstanceIDs, notificationsByUser) } } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() @@ -104,7 +110,13 @@ class NotificationSender extends HouseKeeper { } } - private void sendNotificationsForChangedID(String id, Map heldByToUserSettings, Instant since, Instant until, Set affectedInstanceIDs) { + /** + * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) + * and collect notifications per user into 'notificationsByUser' + */ + private void generateNotificationsForChangedID(String id, Map heldByToUserSettings, + Instant since, Instant until, Set affectedInstanceIDs, + Map> notificationsByUser) { // "versions" come sorted by ascending modification time, so oldest version first. // We want to pick the "from version" (the base for which this notice details changes) // as the last saved version *before* the sought interval. @@ -130,7 +142,8 @@ class NotificationSender extends HouseKeeper { // If we've not already sent a notification for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - sendNotificationsForAffectedInstance(dependerID, heldByToUserSettings, fromVersion, untilVersion, until) + generateNotificationsForAffectedInstance(dependerID, heldByToUserSettings, fromVersion, + untilVersion, until, notificationsByUser) } } } @@ -138,11 +151,12 @@ class NotificationSender extends HouseKeeper { } /** - * Send notices for a bibliographic instance. Beware: fromVersion and untilVersion may not be + * Generate notifications for an affected bibliographic instance. Beware: fromVersion and untilVersion may not be * _of this document_ (id), but rather of a document this instance depends on! */ - private void sendNotificationsForAffectedInstance(String id, Map heldByToUserSettings, DocumentVersion fromVersion, - DocumentVersion untilVersion, Instant creationTime) { + private void generateNotificationsForAffectedInstance(String id, Map heldByToUserSettings, DocumentVersion fromVersion, + DocumentVersion untilVersion, Instant creationTime, + Map> notificationsByUser) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { List users = (List) heldByToUserSettings[library] @@ -159,14 +173,15 @@ class NotificationSender extends HouseKeeper { "https://id.kb.se/changenote/primarytitle" ] } - ] + ], + "email": "noreply@kb.se" }*/ List triggered = changeMatchesAnyTrigger( id, fromVersion.versionWriteTime.toInstant(), creationTime, user, library) if (triggered) { - System.err.println("\tSEND NOTICE FOR USER " + user["id"].toString() + " : " + triggered) + System.err.println("\tSEND NOTICE FOR USER " + user.notificationEmail + " : " + triggered + " on instance: " + id) } } } From bb2b4fc964053900e7d40c15e02d5bba2365cdac Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 3 Oct 2023 15:35:57 +0200 Subject: [PATCH 30/58] Cleaning up a bit --- .../housekeeping/NotificationSender.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index f3d0857950..d733d30fe9 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -102,6 +102,7 @@ class NotificationSender extends HouseKeeper { generateNotificationsForChangedID(id, heldByToUserSettings, from.toInstant(), until.toInstant(), affectedInstanceIDs, notificationsByUser) } + System.err.println("Email summaries to send:\n" + notificationsByUser) } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() throw e @@ -142,8 +143,8 @@ class NotificationSender extends HouseKeeper { // If we've not already sent a notification for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - generateNotificationsForAffectedInstance(dependerID, heldByToUserSettings, fromVersion, - untilVersion, until, notificationsByUser) + generateNotificationsForAffectedInstance(dependerID, heldByToUserSettings, fromVersion.versionWriteTime.toInstant(), + until, notificationsByUser) } } } @@ -154,8 +155,8 @@ class NotificationSender extends HouseKeeper { * Generate notifications for an affected bibliographic instance. Beware: fromVersion and untilVersion may not be * _of this document_ (id), but rather of a document this instance depends on! */ - private void generateNotificationsForAffectedInstance(String id, Map heldByToUserSettings, DocumentVersion fromVersion, - DocumentVersion untilVersion, Instant creationTime, + private void generateNotificationsForAffectedInstance(String id, Map heldByToUserSettings, Instant from, + Instant until, Map> notificationsByUser) { List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { @@ -178,10 +179,13 @@ class NotificationSender extends HouseKeeper { }*/ List triggered = changeMatchesAnyTrigger( - id, fromVersion.versionWriteTime.toInstant(), - creationTime, user, library) + id, from, + until, user, library) if (triggered) { - System.err.println("\tSEND NOTICE FOR USER " + user.notificationEmail + " : " + triggered + " on instance: " + id) + if (!notificationsByUser.containsKey(user.notificationEmail)) + notificationsByUser.put((String)user.notificationEmail, []) + notificationsByUser[user.notificationEmail].add( + "En instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(id) + "\n\tÄndrinskatergorier: " + triggered+"\n") } } } From a6bcf26ff57bf85e29682ddfc4ff8349a37e98d8 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 4 Oct 2023 13:20:27 +0200 Subject: [PATCH 31/58] Add a state table, and use it for rembering the email sending state --- .../housekeeping/NotificationSender.groovy | 34 +++++------- .../migrations/00000021-add-state-table.plsql | 33 ++++++++++++ .../component/PostgreSQLComponent.groovy | 52 +++++++++++++++++++ 3 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 librisxl-tools/postgresql/migrations/00000021-add-state-table.plsql diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index d733d30fe9..c31a9a70fc 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -12,6 +12,8 @@ import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.Timestamp import java.time.Instant +import java.time.ZoneId +import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit @@ -20,6 +22,7 @@ import java.time.temporal.ChronoUnit @Log class NotificationSender extends HouseKeeper { + private final String STATE_KEY = "Email notifications" private String status = "OK" private Whelk whelk @@ -59,7 +62,12 @@ class NotificationSender extends HouseKeeper { } } - //System.err.println("Users map:\n\n" + heldByToUserSettings + "\n\n") + // Determine the time interval of changes for which to generate notifications. + Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. + Map state = whelk.getStorage().getState(STATE_KEY) + if (state && state.lastEmailTime) + from = Timestamp.from( ZonedDateTime.parse( (String) state.lastEmailTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) + Timestamp until = Timestamp.from(Instant.now()) Connection connection PreparedStatement statement @@ -68,24 +76,7 @@ class NotificationSender extends HouseKeeper { connection = whelk.getStorage().getOuterConnection() connection.setAutoCommit(false) try { - // Determine the time interval of changes for which to generate notices. - // This interval, should generally be: From the last generated notice until now. - // However, if there are no previously generated notices (near enough in time), use - // now - [some pre set value], to avoid scanning the whole catalog. - - /*String sql = "SELECT MAX(created) FROM lddb__notifications;" - statement = connection.prepareStatement(sql) - resultSet = statement.executeQuery() - Timestamp from = Timestamp.from(Instant.now().minus(DAYS_TO_KEEP_NOTIFICATIONS, ChronoUnit.DAYS)) - if (resultSet.next()) { - Timestamp lastCreated = resultSet.getTimestamp(1) - if (lastCreated && lastCreated.after(from)) - from = lastCreated - }*/ - Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) - Timestamp until = Timestamp.from(Instant.now()) - - // Then fetch all changed IDs within that interval + // Fetch all changed IDs within the interval String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) @@ -108,6 +99,9 @@ class NotificationSender extends HouseKeeper { throw e } finally { connection.close() + Map newState = new HashMap() + newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() //.atZone(ZoneId.of("UTC")) //ZonedDateTime.from(until.toInstant()).toString() + whelk.getStorage().putState(STATE_KEY, newState) } } @@ -283,7 +277,7 @@ class NotificationSender extends HouseKeeper { return true if (!changeNote.atTime) return true - Instant atTime = ZonedDateTime.parse( (String) changeNote.atTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant(); + Instant atTime = ZonedDateTime.parse( (String) changeNote.atTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() if (atTime.isBefore(before) || atTime.isAfter(after)) return true return false diff --git a/librisxl-tools/postgresql/migrations/00000021-add-state-table.plsql b/librisxl-tools/postgresql/migrations/00000021-add-state-table.plsql new file mode 100644 index 0000000000..9e90233308 --- /dev/null +++ b/librisxl-tools/postgresql/migrations/00000021-add-state-table.plsql @@ -0,0 +1,33 @@ +BEGIN; + +DO $$DECLARE + -- THESE MUST BE CHANGED WHEN YOU COPY THE SCRIPT! + + -- The version you expect the database to have _before_ the migration + old_version numeric := 20; + -- The version the database should have _after_ the migration + new_version numeric := 21; + + -- hands off + existing_version numeric; + +BEGIN + + -- Check existing version +SELECT version from lddb__schema INTO existing_version; +IF ( existing_version <> old_version) THEN + RAISE EXCEPTION 'ASKED TO MIGRATE FROM INCORRECT EXISTING VERSION!'; +ROLLBACK; +END IF; +UPDATE lddb__schema SET version = new_version; + +-- ACTUAL SCHEMA CHANGES HERE: + +CREATE TABLE IF NOT EXISTS lddb__state ( + key text PRIMARY KEY, + value jsonb NOT NULL +); + +END$$; + +COMMIT; diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index ac04872292..3382329db9 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -414,6 +414,16 @@ class PostgreSQLComponent { SELECT id FROM lddb WHERE data#>'{@graph,0,inDataset}' @> ?::jsonb AND deleted = false """.stripIndent() + private static final String GET_STATE = + "SELECT value FROM lddb__state WHERE key = ?" + + private static final String UPSERT_STATE = """ + INSERT INTO lddb__state (key, value) + VALUES (?, ?) + ON CONFLICT (key) DO UPDATE + SET (key, value) = (EXCLUDED.key, EXCLUDED.value) + """.stripIndent() + private static final String GET_USER_DATA = "SELECT data FROM lddb__user_data WHERE id = ?" @@ -2677,6 +2687,48 @@ class PostgreSQLComponent { } } + void putState(String key, Map value) { + withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + try { + PGobject jsonb = new PGobject() + jsonb.setType("jsonb") + jsonb.setValue( mapper.writeValueAsString(value) ) + + preparedStatement = connection.prepareStatement(UPSERT_STATE) + preparedStatement.setString(1, key) + preparedStatement.setObject(2, jsonb) + + preparedStatement.executeUpdate() + } finally { + close(preparedStatement) + } + } + } + + Map getState(String key) { + return withDbConnection { + Connection connection = getMyConnection() + PreparedStatement preparedStatement = null + ResultSet rs = null + try { + preparedStatement = connection.prepareStatement(GET_STATE) + preparedStatement.setString(1, key) + + rs = preparedStatement.executeQuery() + if (rs.next()) { + return mapper.readValue(rs.getString("value"), Map) + } + else { + return null + } + } finally { + close(rs, preparedStatement) + } + } + } + /** * Returns the user-data map for each user _with the user id_ also inserted into the map. */ From bf2375a48a0634bd324f48e93f7eb7ca1e918bb7 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 5 Oct 2023 14:52:46 +0200 Subject: [PATCH 32/58] Add mailing code. --- housekeeping/build.gradle | 3 ++ .../housekeeping/NotificationSender.groovy | 51 +++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/housekeeping/build.gradle b/housekeeping/build.gradle index c4cfc09e6a..fcf0e016b0 100755 --- a/housekeeping/build.gradle +++ b/housekeeping/build.gradle @@ -38,6 +38,9 @@ dependencies { //implementation 'org.apache.commons:commons-lang3:3.3.2' implementation "org.codehaus.groovy:groovy:${groovyVersion}" + + // Email + implementation group: 'org.simplejavamail', name: 'simple-java-mail', version: '8.2.0' } gretty { diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index c31a9a70fc..dfe16e88d4 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -1,18 +1,22 @@ package whelk.housekeeping +import org.simplejavamail.api.email.Email +import org.simplejavamail.api.mailer.Mailer +import org.simplejavamail.email.EmailBuilder +import org.simplejavamail.mailer.MailerBuilder import whelk.Document import whelk.JsonLd import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log import whelk.history.DocumentVersion +import whelk.util.PropertyLoader import java.sql.Connection import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.Timestamp import java.time.Instant -import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter @@ -24,10 +28,26 @@ class NotificationSender extends HouseKeeper { private final String STATE_KEY = "Email notifications" private String status = "OK" - private Whelk whelk + private final Whelk whelk + private final Mailer mailer + private final String senderAddress public NotificationSender(Whelk whelk) { this.whelk = whelk + Properties props = PropertyLoader.loadProperties("secret") + if (props.containsKey("smtpServer") && + props.containsKey("smtpPort") && + props.containsKey("smtpSender") && + props.containsKey("smtpUser") && + props.containsKey("smtpPassword")) + mailer = MailerBuilder + .withSMTPServer( + (String) props.get("smtpServer"), + Integer.parseInt((String)props.get("smtpPort")), + (String) props.get("smtpUser"), + (String) props.get("smtpPassword") + ).buildMailer() + senderAddress = props.get("smtpSender") } public String getName() { @@ -93,18 +113,41 @@ class NotificationSender extends HouseKeeper { generateNotificationsForChangedID(id, heldByToUserSettings, from.toInstant(), until.toInstant(), affectedInstanceIDs, notificationsByUser) } - System.err.println("Email summaries to send:\n" + notificationsByUser) + + //System.err.println("Email summaries to send: \n" + notificationsByUser) + notificationsByUser.keySet().each { email -> + List notifications = notificationsByUser.get(email) + sendEmail(senderAddress, email, "Förändingsmeddelande", notifications.join("\n")) + } + + } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() throw e } finally { connection.close() Map newState = new HashMap() - newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() //.atZone(ZoneId.of("UTC")) //ZonedDateTime.from(until.toInstant()).toString() + newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() whelk.getStorage().putState(STATE_KEY, newState) } } + private void sendEmail(String sender, String recipient, String subject, String body) { + Email email = EmailBuilder.startingBlank() + .to(recipient) + .withSubject(subject) + .from(sender) + .withPlainText(body) + .buildEmail() + + if (mailer) { + log.info("Sending notification (cxz) email to " + recipient) + mailer.sendMail(email) + } else { + log.info("Should now have sent notification (cxz) email to " + recipient + " but SMTP not configured.") + } + } + /** * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) * and collect notifications per user into 'notificationsByUser' From 6e7910457f90fec4537669044bd37c52014fb30a Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 6 Oct 2023 15:04:37 +0200 Subject: [PATCH 33/58] Remove unneeded code and clean up. --- .../housekeeping/NotificationSender.groovy | 26 +++---------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index dfe16e88d4..0069e8379c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -59,14 +59,13 @@ class NotificationSender extends HouseKeeper { } public void trigger() { - // Build a multi-map of library -> list of settings objects for that library's users Map> heldByToUserSettings = new HashMap<>(); { List allUserSettingStrings = whelk.getStorage().getAllUserData() for (Map settings : allUserSettingStrings) { if (!settings["notificationEmail"]) - return + continue settings?.requestedNotifications.each { request -> if (!request instanceof Map) return @@ -114,7 +113,6 @@ class NotificationSender extends HouseKeeper { until.toInstant(), affectedInstanceIDs, notificationsByUser) } - //System.err.println("Email summaries to send: \n" + notificationsByUser) notificationsByUser.keySet().each { email -> List notifications = notificationsByUser.get(email) sendEmail(senderAddress, email, "Förändingsmeddelande", notifications.join("\n")) @@ -139,7 +137,7 @@ class NotificationSender extends HouseKeeper { .from(sender) .withPlainText(body) .buildEmail() - + if (mailer) { log.info("Sending notification (cxz) email to " + recipient) mailer.sendMail(email) @@ -153,24 +151,8 @@ class NotificationSender extends HouseKeeper { * and collect notifications per user into 'notificationsByUser' */ private void generateNotificationsForChangedID(String id, Map heldByToUserSettings, - Instant since, Instant until, Set affectedInstanceIDs, + Instant from, Instant until, Set affectedInstanceIDs, Map> notificationsByUser) { - // "versions" come sorted by ascending modification time, so oldest version first. - // We want to pick the "from version" (the base for which this notice details changes) - // as the last saved version *before* the sought interval. - DocumentVersion fromVersion = null - List versions = whelk.getStorage().loadDocumentHistory(id) - for (DocumentVersion version : versions) { - if (version.doc.getModifiedTimestamp().isBefore(since)) - fromVersion = version - } - if (fromVersion == null) - return - - DocumentVersion untilVersion = versions.last() - if (untilVersion == fromVersion) - return - List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! dependers.each { @@ -180,7 +162,7 @@ class NotificationSender extends HouseKeeper { // If we've not already sent a notification for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - generateNotificationsForAffectedInstance(dependerID, heldByToUserSettings, fromVersion.versionWriteTime.toInstant(), + generateNotificationsForAffectedInstance(dependerID, heldByToUserSettings, from, until, notificationsByUser) } } From de6fe201144f111d4a490e9c01455db7f9420f21 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 10 Oct 2023 10:55:02 +0200 Subject: [PATCH 34/58] Cleanup, and add missing file. --- .../housekeeping/NotificationSender.groovy | 29 ++++++++++++++++--- housekeeping/src/main/resources/log4j2.xml | 22 ++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 housekeeping/src/main/resources/log4j2.xml diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 0069e8379c..ee93c5ba2b 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -9,7 +9,6 @@ import whelk.JsonLd import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log -import whelk.history.DocumentVersion import whelk.util.PropertyLoader import java.sql.Connection @@ -280,9 +279,31 @@ class NotificationSender extends HouseKeeper { case "https://id.kb.se/changenote/serialrelation": historicEmbellish(instanceAfterChange, ["mainEntity", "precededBy", "succeededBy"], after, changeNotes) break - /*case "https://id.kb.se/changenote/primarycontribution": { + case "https://id.kb.se/changenote/primarycontribution": { + Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) + historicEmbellish(instanceBeforeChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) historicEmbellish(instanceAfterChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) - }*/ + Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) + Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) + if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || ! contributionsAfter instanceof List) + break + + for (Object contrBefore : contributionsBefore) { + for (Object contrAfter : contributionsAfter) { + if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution") ) { + if ( contributionsBefore["agent"] != null && contributionsAfter["agent"] != null) { + if ( + contributionsBefore["agent"]["familyName"] != contributionsAfter["agent"]["familyName"] || + contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || + contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] + ) + return true + } + } + } + } + break + } } boolean matches = false @@ -318,7 +339,7 @@ class NotificationSender extends HouseKeeper { * This function mutates docToEmbellish * This function also collects metadata ChangeNotes from embellished records. */ - private historicEmbellish(Document docToEmbellish, List properties, Instant asOf, List changeNotes) { + private void historicEmbellish(Document docToEmbellish, List properties, Instant asOf, List changeNotes) { List graphListToEmbellish = docToEmbellish.data["@graph"] Set alreadyLoadedURIs = [] diff --git a/housekeeping/src/main/resources/log4j2.xml b/housekeeping/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..5819dcd87b --- /dev/null +++ b/housekeeping/src/main/resources/log4j2.xml @@ -0,0 +1,22 @@ + + + + . + + + + + + + + + + + + + + + + + + From de80237e8b58f970742e734a5e9fcee3468a4248 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 10 Oct 2023 13:23:30 +0200 Subject: [PATCH 35/58] Remove changes that are no longer needed. --- .../groovy/whelk/importer/MergeSpec.groovy | 49 +++++++++---------- .../component/PostgreSQLComponent.groovy | 4 +- .../groovy/whelk/history/DocumentVersion.java | 4 +- .../groovy/whelk/history/HistorySpec.groovy | 8 +-- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy index c99d331328..a6a61870ff 100644 --- a/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy +++ b/batchimport/src/test/groovy/whelk/importer/MergeSpec.groovy @@ -6,9 +6,6 @@ import whelk.JsonLd import whelk.history.DocumentVersion import whelk.history.History -import java.sql.Timestamp -import java.time.Instant - class MergeSpec extends Specification { static final Map CONTEXT_DATA = [ @@ -58,7 +55,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -104,7 +101,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -150,7 +147,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -188,7 +185,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) @@ -227,7 +224,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -265,7 +262,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -306,7 +303,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -351,7 +348,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) @@ -398,7 +395,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -444,7 +441,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -482,7 +479,7 @@ class MergeSpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) @@ -536,7 +533,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -617,7 +614,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -698,7 +695,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -779,7 +776,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -885,7 +882,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -989,7 +986,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1081,7 +1078,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1155,7 +1152,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1218,7 +1215,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1286,7 +1283,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1359,7 +1356,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) @@ -1422,7 +1419,7 @@ class MergeSpec extends Specification { ] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) def incoming = new Document( (Map) diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 3382329db9..2f4b41d0f4 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -189,7 +189,7 @@ class PostgreSQLComponent { """.stripIndent() private static final String GET_ALL_DOCUMENT_VERSIONS = """ - SELECT id, data, deleted, created, modified, changedBy, changedIn, pk, GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) as modTime + SELECT id, data, deleted, created, modified, changedBy, changedIn FROM lddb__versions WHERE id = ? ORDER BY GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) ASC @@ -2487,7 +2487,7 @@ class PostgreSQLComponent { while (rs.next()) { def doc = assembleDocument(rs) doc.version = v++ - docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"), rs.getTimestamp("modTime"))) + docList.add(new DocumentVersion(doc, rs.getString("changedBy"), rs.getString("changedIn"))) } } finally { close(rs, selectstmt) diff --git a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java index 037f57fffa..60bdbcc948 100644 --- a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java +++ b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java @@ -12,11 +12,9 @@ public class DocumentVersion { public String changedBy; public String changedIn; - public Timestamp versionWriteTime; - public DocumentVersion(Document doc, String changedBy, String changedIn, Timestamp versionWriteTime) { + public DocumentVersion(Document doc, String changedBy, String changedIn) { this.doc = doc; this.changedBy = changedBy; this.changedIn = changedIn; - this.versionWriteTime = versionWriteTime; } } diff --git a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy index 6bac279500..234b19b686 100644 --- a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy +++ b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy @@ -691,7 +691,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) @@ -726,7 +726,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) @@ -761,7 +761,7 @@ class HistorySpec extends Specification { ]] ] ].collect { change -> - new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn, Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(change.data), change.changedBy, change.changedIn) } def history = new History(versions, ld) @@ -779,7 +779,7 @@ class HistorySpec extends Specification { def ld = new JsonLd(JsonLdSpec.CONTEXT_DATA, display, JsonLdSpec.VOCAB_DATA) def v = versions.collect { data -> - new DocumentVersion(new Document(data), '', '', Timestamp.from(Instant.EPOCH)) + new DocumentVersion(new Document(data), '', '') } def history = new History(v, ld) From b264871775cd7e3a62b91f7834738294bcef0090 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 10 Oct 2023 13:28:01 +0200 Subject: [PATCH 36/58] Cleaning up. --- rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy | 6 ++---- .../main/groovy/whelk/component/PostgreSQLComponent.groovy | 2 +- .../src/main/groovy/whelk/history/DocumentVersion.java | 3 --- whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy | 3 --- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy index f04ffd5bcb..2f505f80e2 100644 --- a/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy +++ b/rest/src/test/groovy/whelk/rest/api/CrudSpec.groovy @@ -20,8 +20,6 @@ import javax.servlet.ServletOutputStream import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse import javax.servlet.http.HttpServletResponseWrapper -import java.sql.Timestamp -import java.time.Instant import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND @@ -819,8 +817,8 @@ class CrudSpec extends Specification { } storage.loadDocumentHistory(_) >> { [ - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", "", Timestamp.from(Instant.EPOCH)), - new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", "", Timestamp.from(Instant.EPOCH)), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'x']]]), "foo", ""), + new DocumentVersion(new Document(['@graph': [['modified':'2022-02-02T12:00:00Z'], ['a': 'y']]]), "bar", ""), ] } diff --git a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy index 2f4b41d0f4..569957d17a 100644 --- a/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy +++ b/whelk-core/src/main/groovy/whelk/component/PostgreSQLComponent.groovy @@ -189,7 +189,7 @@ class PostgreSQLComponent { """.stripIndent() private static final String GET_ALL_DOCUMENT_VERSIONS = """ - SELECT id, data, deleted, created, modified, changedBy, changedIn + SELECT id, data, deleted, created, modified, changedBy, changedIn FROM lddb__versions WHERE id = ? ORDER BY GREATEST(modified, (data#>>'{@graph,0,generationDate}')::timestamptz) ASC diff --git a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java index 60bdbcc948..9f9f68384f 100644 --- a/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java +++ b/whelk-core/src/main/groovy/whelk/history/DocumentVersion.java @@ -2,8 +2,6 @@ import whelk.Document; -import java.sql.Timestamp; - /** * Represents a version of a record, including the out-of-record info (like the changedBy column) */ @@ -11,7 +9,6 @@ public class DocumentVersion { public Document doc; public String changedBy; public String changedIn; - public DocumentVersion(Document doc, String changedBy, String changedIn) { this.doc = doc; this.changedBy = changedBy; diff --git a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy index 234b19b686..2ed0826f3c 100644 --- a/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy +++ b/whelk-core/src/test/groovy/whelk/history/HistorySpec.groovy @@ -7,9 +7,6 @@ import whelk.JsonLd import whelk.util.Jackson import whelk.util.JsonLdSpec -import java.sql.Timestamp -import java.time.Instant - class HistorySpec extends Specification { def "array(set) order does not matter"() { given: From efe5c5298e855cc12a7abbb59f5ab6840223d1ce Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 10 Oct 2023 14:02:36 +0200 Subject: [PATCH 37/58] Slightly improved email contents. --- .../housekeeping/NotificationSender.groovy | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index ee93c5ba2b..69747cd096 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -130,14 +130,14 @@ class NotificationSender extends HouseKeeper { } private void sendEmail(String sender, String recipient, String subject, String body) { - Email email = EmailBuilder.startingBlank() - .to(recipient) - .withSubject(subject) - .from(sender) - .withPlainText(body) - .buildEmail() - if (mailer) { + Email email = EmailBuilder.startingBlank() + .to(recipient) + .withSubject(subject) + .from(sender) + .withPlainText(body) + .buildEmail() + log.info("Sending notification (cxz) email to " + recipient) mailer.sendMail(email) } else { @@ -187,7 +187,7 @@ class NotificationSender extends HouseKeeper { "id": "sldknfslkdnsdlkgnsdkjgnb" "requestedNotifications": [ { - "library": "https://libris.kb.se/library/Utb1", + "heldBy": "https://libris.kb.se/library/Utb1", "triggers": [ "https://id.kb.se/changenote/primarytitle" ] @@ -202,21 +202,42 @@ class NotificationSender extends HouseKeeper { if (triggered) { if (!notificationsByUser.containsKey(user.notificationEmail)) notificationsByUser.put((String)user.notificationEmail, []) - notificationsByUser[user.notificationEmail].add( - "En instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(id) + "\n\tÄndrinskatergorier: " + triggered+"\n") + notificationsByUser[user.notificationEmail].add(generateEmailBody(id, triggered)) } } } } } + private String generateEmailBody(String changedInstanceId, List triggeredCategories) { + Document changed = whelk.getStorage().load(changedInstanceId) + String mainTitle = Document._get(["@graph", 1, "hasTitle", 0, "mainTitle"], changed.data) + if (mainTitle == null) + mainTitle = "Ingen huvudtitel" + + String changeCategories = "" + for (String categoryUri : triggeredCategories) { + Document categoryDoc = whelk.getStorage().loadDocumentByMainId(categoryUri) + String swedishLabel = Document._get(["@graph", 1, "prefLabelByLand", "sv"], categoryDoc.data) + if (swedishLabel) { + changeCategories += swedishLabel + "(" + categoryUri + ") " + } else { + changeCategories += categoryUri + " " + } + } + + return "Instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(changedInstanceId) + + "(" + mainTitle + ")\n\t" + + "Ändrinskatergorier: " + changeCategories + "\n" + } + /** * 'from' and 'until' are the instants of writing for the changed record (the previous and current versions) * 'user' is the user-data map for a user (which includes their selection of triggers). * 'library' is a library ("sigel") holding the instance in question. * * This function answers the question: Has 'user' requested to be notified of the occurred changes between - * 'from' and 'until' for instances held by 'library'? + * 'from' and 'until' for instances held by 'heldBy'? * * Returns the URIs of all triggered rules/triggers. */ From 52fc4eed96544c729d6920143a459d298412838c Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 12 Oct 2023 11:02:42 +0200 Subject: [PATCH 38/58] Play politics. --- .../groovy/whelk/housekeeping/NotificationSender.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 69747cd096..28d903ec92 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -301,9 +301,13 @@ class NotificationSender extends HouseKeeper { historicEmbellish(instanceAfterChange, ["mainEntity", "precededBy", "succeededBy"], after, changeNotes) break case "https://id.kb.se/changenote/primarycontribution": { + historicEmbellish(instanceAfterChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) + + // Uncomment this block, to get the full functionality. The feature had to be hamstrung due to political + // reasons. + /* Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) historicEmbellish(instanceBeforeChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) - historicEmbellish(instanceAfterChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || ! contributionsAfter instanceof List) @@ -322,7 +326,7 @@ class NotificationSender extends HouseKeeper { } } } - } + }*/ break } } From fde1882d67b8b8fd7211b289352b52125544897b Mon Sep 17 00:00:00 2001 From: Jannis Mohlin Tsiroyannis Date: Tue, 17 Oct 2023 09:12:52 +0200 Subject: [PATCH 39/58] Update housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy Co-authored-by: Anders Jensen-Urstad --- .../main/groovy/whelk/housekeeping/NotificationSender.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 28d903ec92..4fb29d2f18 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -65,7 +65,7 @@ class NotificationSender extends HouseKeeper { for (Map settings : allUserSettingStrings) { if (!settings["notificationEmail"]) continue - settings?.requestedNotifications.each { request -> + settings?.requestedNotifications?.each { request -> if (!request instanceof Map) return if (!request["heldBy"]) From 35c3608b4849611f5e10c130f88492207ad55413 Mon Sep 17 00:00:00 2001 From: Jannis Mohlin Tsiroyannis Date: Tue, 17 Oct 2023 09:17:41 +0200 Subject: [PATCH 40/58] Update housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy Co-authored-by: Anders Jensen-Urstad --- .../main/groovy/whelk/housekeeping/NotificationSender.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 4fb29d2f18..ad64a0bab2 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -74,8 +74,7 @@ class NotificationSender extends HouseKeeper { String heldBy = request["heldBy"] if (!heldByToUserSettings.containsKey(heldBy)) heldByToUserSettings.put(heldBy, []) - List userSettingsForThisHeldBy = heldByToUserSettings[heldBy] - userSettingsForThisHeldBy.add(settings) + heldByToUserSettings[heldBy].add(settings) } } } From c7069e75cd9431b0facb30d03d3fcdcce1afaa5a Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Tue, 17 Oct 2023 10:15:54 +0200 Subject: [PATCH 41/58] Minor email body adjustment. --- .../main/groovy/whelk/housekeeping/NotificationSender.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index ad64a0bab2..4e90b14c1b 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -219,7 +219,7 @@ class NotificationSender extends HouseKeeper { Document categoryDoc = whelk.getStorage().loadDocumentByMainId(categoryUri) String swedishLabel = Document._get(["@graph", 1, "prefLabelByLand", "sv"], categoryDoc.data) if (swedishLabel) { - changeCategories += swedishLabel + "(" + categoryUri + ") " + changeCategories += swedishLabel + " " } else { changeCategories += categoryUri + " " } @@ -227,7 +227,7 @@ class NotificationSender extends HouseKeeper { return "Instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(changedInstanceId) + "(" + mainTitle + ")\n\t" + - "Ändrinskatergorier: " + changeCategories + "\n" + "Ändring har skett med avseende på: " + changeCategories + "\n\n" } /** From 3e5df42b7919f6813cc6805de7eed63851c7bc01 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 27 Oct 2023 12:16:52 +0200 Subject: [PATCH 42/58] Begin the process of moving from CXZ design 2 to CXZ desgin 3. --- .../housekeeping/NotificationGenerator.groovy | 213 +++++++++ .../housekeeping/NotificationSender.groovy | 428 ------------------ .../whelk/housekeeping/WebInterface.groovy | 2 +- 3 files changed, 214 insertions(+), 429 deletions(-) create mode 100644 housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy delete mode 100644 housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy new file mode 100644 index 0000000000..8e7be6dea6 --- /dev/null +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -0,0 +1,213 @@ +package whelk.housekeeping + +import org.simplejavamail.api.email.Email +import org.simplejavamail.api.mailer.Mailer +import org.simplejavamail.email.EmailBuilder +import org.simplejavamail.mailer.MailerBuilder +import whelk.Document +import whelk.JsonLd +import whelk.Whelk +import groovy.transform.CompileStatic +import groovy.util.logging.Log4j2 as Log +import whelk.util.PropertyLoader + +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.Timestamp +import java.time.Instant +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +@CompileStatic +@Log +class NotificationGenerator extends HouseKeeper { + + private final String STATE_KEY = "Email notifications" + private String status = "OK" + private final Whelk whelk + + public NotificationGenerator(Whelk whelk) { + this.whelk = whelk + } + + public String getName() { + return "Notifications generator" + } + + public String getStatusDescription() { + return status + } + + public void trigger() { + // Determine the time interval of changes for which to generate notifications. + Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. + Map state = whelk.getStorage().getState(STATE_KEY) + if (state && state.lastEmailTime) + from = Timestamp.from( ZonedDateTime.parse( (String) state.lastEmailTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) + Timestamp until = Timestamp.from(Instant.now()) + + Connection connection + PreparedStatement statement + ResultSet resultSet + + connection = whelk.getStorage().getOuterConnection() + connection.setAutoCommit(false) + try { + // Fetch all changed IDs within the interval + String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" + connection.setAutoCommit(false) + statement = connection.prepareStatement(sql) + statement.setTimestamp(1, from) + statement.setTimestamp(2, until) + statement.setFetchSize(512) + resultSet = statement.executeQuery() + // If both an instance and one of it's dependencies are affected within the same interval, we will + // (without this check) try to generate notifications for said instance twice. + Set affectedInstanceIDs = [] + while (resultSet.next()) { + String id = resultSet.getString("id") + generateNotificationsForChangedID(id, from.toInstant(), + until.toInstant(), affectedInstanceIDs) + } + } catch (Throwable e) { + status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() + throw e + } finally { + connection.close() + Map newState = new HashMap() + newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() + whelk.getStorage().putState(STATE_KEY, newState) + } + } + + /** + * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) + * relevant notification-records + */ + private void generateNotificationsForChangedID(String id, Instant from, Instant until, Set affectedInstanceIDs) { + List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) + dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! + dependers.each { + String dependerID = it[0] + String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) + if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { + // If we've not already sent a notification for this instance! + if (!affectedInstanceIDs.contains(dependerID)) { + affectedInstanceIDs.add(dependerID) + generateNotificationsForAffectedInstance(dependerID, from, until) + } + } + } + } + + private void generateNotificationsForAffectedInstance(String instanceId, Instant before, Instant after) { + List propertiesToEmbellish = [ + "mainEntity", + "instanceOf", + "contribution", + "hasTitle", + "intendedAudience", + "classification", + "precededBy", + "succeededBy", + "contribution", + ] + Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(after)) + historicEmbellish(instanceAfterChange, propertiesToEmbellish, after) + Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) + historicEmbellish(instanceBeforeChange, propertiesToEmbellish, before); + + // Check for primary contribution changes + { + Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) + Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) + if (contributionsBefore != null && contributionsAfter != null && contributionsBefore instanceof List && contributionsAfter instanceof List) { + for (Object contrBefore : contributionsBefore) { + for (Object contrAfter : contributionsAfter) { + if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution")) { + if (contributionsBefore["agent"] != null && contributionsAfter["agent"] != null) { + if ( + contributionsBefore["agent"]["familyName"] != contributionsAfter["agent"]["familyName"] || + contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || + contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] + ) + System.err.println("MAKING PRIM. CONTR. NOTIFICATION!") + } + } + } + } + } + } + } + + /** + * This is a simplified/specialized from of 'embellish', for historic data and using only select properties. + * The full general embellish code can not help us here, because it is based on the idea of cached cards, + * which can (and must!) only cache the latest/current data for each card, which isn't what we need here + * (we need to embellish older historic data). + * + * This function mutates docToEmbellish + */ + private void historicEmbellish(Document docToEmbellish, List properties, Instant asOf) { + List graphListToEmbellish = (List) docToEmbellish.data["@graph"] + Set alreadyLoadedURIs = [] + + for (int i = 0; i < properties.size(); ++i) { + Set uris = findLinkedURIs(graphListToEmbellish, properties) + uris.removeAll(alreadyLoadedURIs) + if (uris.isEmpty()) + break + + Map linkedDocumentsByUri = whelk.bulkLoad(uris, asOf) + linkedDocumentsByUri.each { + List linkedGraphList = (List) it.value.data["@graph"] + if (linkedGraphList.size() > 1) + graphListToEmbellish.add(linkedGraphList[1]) + } + alreadyLoadedURIs.addAll(uris) + } + + docToEmbellish.data = JsonLd.frame(docToEmbellish.getCompleteId(), docToEmbellish.data) + } + + private Set findLinkedURIs(Object node, List properties) { + Set uris = [] + if (node instanceof List) { + for (Object element : node) { + uris.addAll(findLinkedURIs(element, properties)) + } + } + else if (node instanceof Map) { + for (String key : node.keySet()) { + if (properties.contains(key)) { + uris.addAll(getLinkIfAny(node[key])) + } + uris.addAll(findLinkedURIs(node[key], properties)) + } + } + return uris + } + + private List getLinkIfAny(Object node) { + List uris = [] + if (node instanceof Map) { + if (node.containsKey("@id")) { + uris.add((String) node["@id"]) + } + } + if (node instanceof List) { + for (Object element : node) { + if (element instanceof Map) { + if (element.containsKey("@id")) { + uris.add((String) element["@id"]) + } + } + } + } + return uris + } + +} diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy deleted file mode 100644 index 4e90b14c1b..0000000000 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ /dev/null @@ -1,428 +0,0 @@ -package whelk.housekeeping - -import org.simplejavamail.api.email.Email -import org.simplejavamail.api.mailer.Mailer -import org.simplejavamail.email.EmailBuilder -import org.simplejavamail.mailer.MailerBuilder -import whelk.Document -import whelk.JsonLd -import whelk.Whelk -import groovy.transform.CompileStatic -import groovy.util.logging.Log4j2 as Log -import whelk.util.PropertyLoader - -import java.sql.Connection -import java.sql.PreparedStatement -import java.sql.ResultSet -import java.sql.Timestamp -import java.time.Instant -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import java.time.temporal.ChronoUnit - -@CompileStatic -@Log -class NotificationSender extends HouseKeeper { - - private final String STATE_KEY = "Email notifications" - private String status = "OK" - private final Whelk whelk - private final Mailer mailer - private final String senderAddress - - public NotificationSender(Whelk whelk) { - this.whelk = whelk - Properties props = PropertyLoader.loadProperties("secret") - if (props.containsKey("smtpServer") && - props.containsKey("smtpPort") && - props.containsKey("smtpSender") && - props.containsKey("smtpUser") && - props.containsKey("smtpPassword")) - mailer = MailerBuilder - .withSMTPServer( - (String) props.get("smtpServer"), - Integer.parseInt((String)props.get("smtpPort")), - (String) props.get("smtpUser"), - (String) props.get("smtpPassword") - ).buildMailer() - senderAddress = props.get("smtpSender") - } - - public String getName() { - return "Notifications sender" - } - - public String getStatusDescription() { - return status - } - - public void trigger() { - // Build a multi-map of library -> list of settings objects for that library's users - Map> heldByToUserSettings = new HashMap<>(); - { - List allUserSettingStrings = whelk.getStorage().getAllUserData() - for (Map settings : allUserSettingStrings) { - if (!settings["notificationEmail"]) - continue - settings?.requestedNotifications?.each { request -> - if (!request instanceof Map) - return - if (!request["heldBy"]) - return - - String heldBy = request["heldBy"] - if (!heldByToUserSettings.containsKey(heldBy)) - heldByToUserSettings.put(heldBy, []) - heldByToUserSettings[heldBy].add(settings) - } - } - } - - // Determine the time interval of changes for which to generate notifications. - Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. - Map state = whelk.getStorage().getState(STATE_KEY) - if (state && state.lastEmailTime) - from = Timestamp.from( ZonedDateTime.parse( (String) state.lastEmailTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) - Timestamp until = Timestamp.from(Instant.now()) - - Connection connection - PreparedStatement statement - ResultSet resultSet - - connection = whelk.getStorage().getOuterConnection() - connection.setAutoCommit(false) - try { - // Fetch all changed IDs within the interval - String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" - connection.setAutoCommit(false) - statement = connection.prepareStatement(sql) - statement.setTimestamp(1, from) - statement.setTimestamp(2, until) - statement.setFetchSize(512) - resultSet = statement.executeQuery() - // If both an instance and one of it's dependencies are affected within the same interval, we will - // (without this check) try to generate notifications for said instance twice. - Set affectedInstanceIDs = [] - Map> notificationsByUser = new HashMap<>() - while (resultSet.next()) { - String id = resultSet.getString("id") - generateNotificationsForChangedID(id, heldByToUserSettings, from.toInstant(), - until.toInstant(), affectedInstanceIDs, notificationsByUser) - } - - notificationsByUser.keySet().each { email -> - List notifications = notificationsByUser.get(email) - sendEmail(senderAddress, email, "Förändingsmeddelande", notifications.join("\n")) - } - - - } catch (Throwable e) { - status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() - throw e - } finally { - connection.close() - Map newState = new HashMap() - newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() - whelk.getStorage().putState(STATE_KEY, newState) - } - } - - private void sendEmail(String sender, String recipient, String subject, String body) { - if (mailer) { - Email email = EmailBuilder.startingBlank() - .to(recipient) - .withSubject(subject) - .from(sender) - .withPlainText(body) - .buildEmail() - - log.info("Sending notification (cxz) email to " + recipient) - mailer.sendMail(email) - } else { - log.info("Should now have sent notification (cxz) email to " + recipient + " but SMTP not configured.") - } - } - - /** - * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) - * and collect notifications per user into 'notificationsByUser' - */ - private void generateNotificationsForChangedID(String id, Map heldByToUserSettings, - Instant from, Instant until, Set affectedInstanceIDs, - Map> notificationsByUser) { - List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) - dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! - dependers.each { - String dependerID = it[0] - String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) - if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { - // If we've not already sent a notification for this instance! - if (!affectedInstanceIDs.contains(dependerID)) { - affectedInstanceIDs.add(dependerID) - generateNotificationsForAffectedInstance(dependerID, heldByToUserSettings, from, - until, notificationsByUser) - } - } - } - - } - - /** - * Generate notifications for an affected bibliographic instance. Beware: fromVersion and untilVersion may not be - * _of this document_ (id), but rather of a document this instance depends on! - */ - private void generateNotificationsForAffectedInstance(String id, Map heldByToUserSettings, Instant from, - Instant until, - Map> notificationsByUser) { - List libraries = whelk.getStorage().getAllLibrariesHolding(id) - for (String library : libraries) { - List users = (List) heldByToUserSettings[library] - if (users) { - for (Map user : users) { - /* - 'user' is now a map looking something like this: - { - "id": "sldknfslkdnsdlkgnsdkjgnb" - "requestedNotifications": [ - { - "heldBy": "https://libris.kb.se/library/Utb1", - "triggers": [ - "https://id.kb.se/changenote/primarytitle" - ] - } - ], - "email": "noreply@kb.se" - }*/ - - List triggered = changeMatchesAnyTrigger( - id, from, - until, user, library) - if (triggered) { - if (!notificationsByUser.containsKey(user.notificationEmail)) - notificationsByUser.put((String)user.notificationEmail, []) - notificationsByUser[user.notificationEmail].add(generateEmailBody(id, triggered)) - } - } - } - } - } - - private String generateEmailBody(String changedInstanceId, List triggeredCategories) { - Document changed = whelk.getStorage().load(changedInstanceId) - String mainTitle = Document._get(["@graph", 1, "hasTitle", 0, "mainTitle"], changed.data) - if (mainTitle == null) - mainTitle = "Ingen huvudtitel" - - String changeCategories = "" - for (String categoryUri : triggeredCategories) { - Document categoryDoc = whelk.getStorage().loadDocumentByMainId(categoryUri) - String swedishLabel = Document._get(["@graph", 1, "prefLabelByLand", "sv"], categoryDoc.data) - if (swedishLabel) { - changeCategories += swedishLabel + " " - } else { - changeCategories += categoryUri + " " - } - } - - return "Instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(changedInstanceId) + - "(" + mainTitle + ")\n\t" + - "Ändring har skett med avseende på: " + changeCategories + "\n\n" - } - - /** - * 'from' and 'until' are the instants of writing for the changed record (the previous and current versions) - * 'user' is the user-data map for a user (which includes their selection of triggers). - * 'library' is a library ("sigel") holding the instance in question. - * - * This function answers the question: Has 'user' requested to be notified of the occurred changes between - * 'from' and 'until' for instances held by 'heldBy'? - * - * Returns the URIs of all triggered rules/triggers. - */ - private List changeMatchesAnyTrigger(String instanceId, Instant from, Instant until, Map user, String heldBy) { - - List triggeredTriggers = [] - - user.requestedNotifications.each { request -> - - // This stuff (the request) comes from a user, so we must be super paranoid about it being correctly formed. - - if (! request instanceof Map) - return - - if (! request["heldBy"] instanceof String) - return - - if (request["heldBy"] != heldBy) - return - - if (! request["triggers"] instanceof List) - return - - for (Object triggerObject : request["triggers"]) { - if (! triggerObject instanceof String) - return - String triggerUri = (String) triggerObject - if (triggerIsTriggered(instanceId, from, until, triggerUri)) - triggeredTriggers.add(triggerUri) - } - } - - return triggeredTriggers - } - - private boolean triggerIsTriggered(String instanceId, Instant before, Instant after, String triggerUri) { - Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(after)) - - // Populate 'changeNotes' with every note affecting this instance in the specified interval - List changeNotes = new ArrayList<>(); - List notesOnInstance = (List) Document._get(["@graph", 0, "hasChangeNote"], instanceAfterChange.data) - if (notesOnInstance != null) - changeNotes.addAll(notesOnInstance) - switch (triggerUri) { - case "https://id.kb.se/changenote/primarytitle": - case "https://id.kb.se/changenote/maintitle": - historicEmbellish(instanceAfterChange, ["mainEntity", "hasTitle"], after, changeNotes) - break - case "https://id.kb.se/changenote/primarypublication": - case "https://id.kb.se/changenote/serialtermination": - historicEmbellish(instanceAfterChange, ["publication"], after, changeNotes) - break - case "https://id.kb.se/changenote/intendedaudience": - historicEmbellish(instanceAfterChange, ["mainEntity", "intendedAudience"], after, changeNotes) - break - case "https://id.kb.se/changenote/ddcclassification": - case "https://id.kb.se/changenote/sabclassification": - historicEmbellish(instanceAfterChange, ["mainEntity", "classification"], after, changeNotes) - break - case "https://id.kb.se/changenote/serialrelation": - historicEmbellish(instanceAfterChange, ["mainEntity", "precededBy", "succeededBy"], after, changeNotes) - break - case "https://id.kb.se/changenote/primarycontribution": { - historicEmbellish(instanceAfterChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) - - // Uncomment this block, to get the full functionality. The feature had to be hamstrung due to political - // reasons. - /* - Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) - historicEmbellish(instanceBeforeChange, ["mainEntity", "instanceOf", "contribution"], after, changeNotes) - Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) - Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) - if (contributionsBefore == null || contributionsAfter == null || ! contributionsBefore instanceof List || ! contributionsAfter instanceof List) - break - - for (Object contrBefore : contributionsBefore) { - for (Object contrAfter : contributionsAfter) { - if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution") ) { - if ( contributionsBefore["agent"] != null && contributionsAfter["agent"] != null) { - if ( - contributionsBefore["agent"]["familyName"] != contributionsAfter["agent"]["familyName"] || - contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || - contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] - ) - return true - } - } - } - }*/ - break - } - } - - boolean matches = false - filterChangeNotesNotInInterval(changeNotes, before, after) - changeNotes.each { note -> - note.category.each { category -> - if (category["@id"] == triggerUri) - matches = true - } - } - return matches - } - - private void filterChangeNotesNotInInterval(List changeNotes, Instant before, Instant after) { - changeNotes.removeAll { changeNote -> - if (changeNote == null) - return true - if (!changeNote.atTime) - return true - Instant atTime = ZonedDateTime.parse( (String) changeNote.atTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() - if (atTime.isBefore(before) || atTime.isAfter(after)) - return true - return false - } - } - - /** - * This is a simplified/specialized from of 'embellish', for historic data and using only select properties. - * The full general embellish code can not help us here, because it is based on the idea of cached cards, - * which can (and must!) only cache the latest/current data for each card, which isn't what we need here - * (we need to embellish older historic data). - * - * This function mutates docToEmbellish - * This function also collects metadata ChangeNotes from embellished records. - */ - private void historicEmbellish(Document docToEmbellish, List properties, Instant asOf, List changeNotes) { - List graphListToEmbellish = docToEmbellish.data["@graph"] - Set alreadyLoadedURIs = [] - - for (int i = 0; i < properties.size(); ++i) { - Set uris = findLinkedURIs(graphListToEmbellish, properties) - uris.removeAll(alreadyLoadedURIs) - if (uris.isEmpty()) - break - - Map linkedDocumentsByUri = whelk.bulkLoad(uris, asOf) - linkedDocumentsByUri.each { - List linkedGraphList = it.value.data["@graph"] - if (linkedGraphList.size() > 1) - graphListToEmbellish.add(linkedGraphList[1]) - linkedGraphList[0]["hasChangeNote"].each { changeNote -> - changeNotes.add( (Map) changeNote ) - } - } - alreadyLoadedURIs.addAll(uris) - } - - docToEmbellish.data = JsonLd.frame(docToEmbellish.getCompleteId(), docToEmbellish.data) - } - - private Set findLinkedURIs(Object node, List properties) { - Set uris = [] - if (node instanceof List) { - for (Object element : node) { - uris.addAll(findLinkedURIs(element, properties)) - } - } - else if (node instanceof Map) { - for (String key : node.keySet()) { - if (properties.contains(key)) { - uris.addAll(getLinkIfAny(node[key])) - } - uris.addAll(findLinkedURIs(node[key], properties)) - } - } - return uris - } - - private List getLinkIfAny(Object node) { - List uris = [] - if (node instanceof Map) { - if (node.containsKey("@id")) { - uris.add((String) node["@id"]) - } - } - if (node instanceof List) { - for (Object element : node) { - if (element instanceof Map) { - if (element.containsKey("@id")) { - uris.add((String) element["@id"]) - } - } - } - } - return uris - } - -} diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 3d4d50b8da..4040173f6b 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -30,7 +30,7 @@ public class WebInterface extends HttpServlet { Whelk whelk = Whelk.createLoadedCoreWhelk() houseKeepers = [] - houseKeepers.add(new NotificationSender(whelk)) + houseKeepers.add(new NotificationGenerator(whelk)) for (HouseKeeper hk : houseKeepers) { timer.scheduleAtFixedRate({ From 0ae2481d1a412fbd457f57dad120b78eee0f09da Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 27 Oct 2023 14:03:44 +0200 Subject: [PATCH 43/58] Add missing property to embellish. --- .../groovy/whelk/housekeeping/NotificationGenerator.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 8e7be6dea6..335cd03e02 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -114,12 +114,15 @@ class NotificationGenerator extends HouseKeeper { "precededBy", "succeededBy", "contribution", + "agent", ] Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(after)) historicEmbellish(instanceAfterChange, propertiesToEmbellish, after) Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) historicEmbellish(instanceBeforeChange, propertiesToEmbellish, before); + //System.err.println(" ******* NOW SCANNING " + instanceId + " FOR IMPLICIT CHANGES!\n\n"); + // Check for primary contribution changes { Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) From 022a2637c0dcd578476a8c56bd12f6200ffbe122 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 1 Nov 2023 16:04:50 +0100 Subject: [PATCH 44/58] Create ChangeObservation records. --- .../housekeeping/NotificationGenerator.groovy | 86 ++++++++++++++++--- 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 335cd03e02..91f60012d2 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -1,16 +1,14 @@ package whelk.housekeeping -import org.simplejavamail.api.email.Email -import org.simplejavamail.api.mailer.Mailer -import org.simplejavamail.email.EmailBuilder -import org.simplejavamail.mailer.MailerBuilder +import org.apache.jena.ext.com.google.common.collect.Lists import whelk.Document +import whelk.IdGenerator import whelk.JsonLd import whelk.Whelk import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log -import whelk.util.PropertyLoader +import java.sql.Array import java.sql.Connection import java.sql.PreparedStatement import java.sql.ResultSet @@ -21,6 +19,8 @@ import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit +import static whelk.util.Jackson.mapper + @CompileStatic @Log class NotificationGenerator extends HouseKeeper { @@ -57,7 +57,7 @@ class NotificationGenerator extends HouseKeeper { connection.setAutoCommit(false) try { // Fetch all changed IDs within the interval - String sql = "SELECT id FROM lddb WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? );" + String sql = "SELECT id, ARRAY_AGG(data#>>'{@graph,0,hasChangeNote}') as changeNotes FROM lddb__versions WHERE collection IN ('bib', 'auth') AND ( modified > ? AND modified <= ? ) group by id;" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -69,7 +69,17 @@ class NotificationGenerator extends HouseKeeper { Set affectedInstanceIDs = [] while (resultSet.next()) { String id = resultSet.getString("id") - generateNotificationsForChangedID(id, from.toInstant(), + + Array changeNotesArray = resultSet.getArray("changeNotes") + + // There is some groovy type-nonsense going on with the array types, simply doing + // List changeNotes = Arrays.asList( changeNotesArray.getArray() ) won't work. + List changeNotes = [] + for (Object o : changeNotesArray.getArray()) { + changeNotes.add(o) + } + + generateNotificationsForChangedID(id, changeNotes, from.toInstant(), until.toInstant(), affectedInstanceIDs) } } catch (Throwable e) { @@ -87,7 +97,7 @@ class NotificationGenerator extends HouseKeeper { * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) * relevant notification-records */ - private void generateNotificationsForChangedID(String id, Instant from, Instant until, Set affectedInstanceIDs) { + private void generateNotificationsForChangedID(String id, List changeNotes, Instant from, Instant until, Set affectedInstanceIDs) { List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! dependers.each { @@ -97,13 +107,13 @@ class NotificationGenerator extends HouseKeeper { // If we've not already sent a notification for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - generateNotificationsForAffectedInstance(dependerID, from, until) + generateNotificationsForAffectedInstance(dependerID, changeNotes, from, until) } } } } - private void generateNotificationsForAffectedInstance(String instanceId, Instant before, Instant after) { + private void generateNotificationsForAffectedInstance(String instanceId, List changeNotes, Instant before, Instant after) { List propertiesToEmbellish = [ "mainEntity", "instanceOf", @@ -137,7 +147,7 @@ class NotificationGenerator extends HouseKeeper { contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] ) - System.err.println("MAKING PRIM. CONTR. NOTIFICATION!") + makeChangeObservation(instanceId, changeNotes, "https://id.kb.se/changenote/primarycontribution", (Map) contrBefore, (Map) contrAfter) } } } @@ -146,6 +156,60 @@ class NotificationGenerator extends HouseKeeper { } } + private void makeChangeObservation(String instanceId, List changeNotes, String categoryUri, Map oldValue, Map newValue) { + String newId = IdGenerator.generate() + String metadataUri = Document.BASE_URI.toString() + newId + String mainEntityUri = metadataUri+"#it" + + Map observationData = [ "@graph":[ + [ + "@id" : metadataUri, + "@type" : "Record", + "mainEntity" : ["@id" : mainEntityUri], + ], + [ + "@id" : mainEntityUri, + "@type" : "ChangeObservation", + "about" : ["@id" : Document.BASE_URI.toString() + instanceId], + "representationBefore" : oldValue, + "representationAfter" : newValue, + ] + ]] + + List comments = extractComments(changeNotes) + if (comments) { + observationData["@graph"][1]["comment"] = comments + } + + Document observationDocument = new Document(observationData) + + System.err.println(" ** Made change observation for instance: " + instanceId + " , category: " + categoryUri + " , notes: " + changeNotes + + "\n resulting document: " + observationDocument.getDataAsString()) + + if (!whelk.createDocument(observationDocument, "NotificationGenerator", "SEK", "none", false)) { + log.error("Failed to create ChangeObservation for $instanceId ($categoryUri).") + } + } + + private List extractComments(List changeNotes) { + List comments = [] + for (Object changeNote : changeNotes) { + if ( ! (changeNote instanceof String) ) + continue + Map changeNoteMap = mapper.readValue( (String) changeNote, Map) + comments.addAll( asList(changeNoteMap["comment"]) ) + } + return comments + } + + private List asList(Object o) { + if (o == null) + return [] + if (o instanceof List) + return o + return [o] + } + /** * This is a simplified/specialized from of 'embellish', for historic data and using only select properties. * The full general embellish code can not help us here, because it is based on the idea of cached cards, From a75546046eb1306b7c14ec91f827a5a6a238e995 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 2 Nov 2023 12:40:45 +0100 Subject: [PATCH 45/58] Progress towards emailing. --- housekeeping/build.gradle | 4 +++ .../housekeeping/NotificationGenerator.groovy | 19 +++++----- .../whelk/housekeeping/WebInterface.groovy | 35 +++++++++++-------- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/housekeeping/build.gradle b/housekeeping/build.gradle index fcf0e016b0..9239c707df 100755 --- a/housekeeping/build.gradle +++ b/housekeeping/build.gradle @@ -41,6 +41,10 @@ dependencies { // Email implementation group: 'org.simplejavamail', name: 'simple-java-mail', version: '8.2.0' + + // Cron + implementation group: 'it.sauronsoftware.cron4j', name: 'cron4j', version: '2.2.5' + } gretty { diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 91f60012d2..04b92cc6bd 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -1,6 +1,5 @@ package whelk.housekeeping -import org.apache.jena.ext.com.google.common.collect.Lists import whelk.Document import whelk.IdGenerator import whelk.JsonLd @@ -25,7 +24,7 @@ import static whelk.util.Jackson.mapper @Log class NotificationGenerator extends HouseKeeper { - private final String STATE_KEY = "Email notifications" + public static final String STATE_KEY = "CXZ notification generator" private String status = "OK" private final Whelk whelk @@ -41,12 +40,16 @@ class NotificationGenerator extends HouseKeeper { return status } + public String getCronSchedule() { + return "* * * * *" + } + public void trigger() { // Determine the time interval of changes for which to generate notifications. Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. Map state = whelk.getStorage().getState(STATE_KEY) - if (state && state.lastEmailTime) - from = Timestamp.from( ZonedDateTime.parse( (String) state.lastEmailTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) + if (state && state.lastGenerationTime) + from = Timestamp.from( ZonedDateTime.parse( (String) state.lastGenerationTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) Timestamp until = Timestamp.from(Instant.now()) Connection connection @@ -88,7 +91,7 @@ class NotificationGenerator extends HouseKeeper { } finally { connection.close() Map newState = new HashMap() - newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() + newState.lastGenerationTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() whelk.getStorage().putState(STATE_KEY, newState) } } @@ -131,8 +134,6 @@ class NotificationGenerator extends HouseKeeper { Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) historicEmbellish(instanceBeforeChange, propertiesToEmbellish, before); - //System.err.println(" ******* NOW SCANNING " + instanceId + " FOR IMPLICIT CHANGES!\n\n"); - // Check for primary contribution changes { Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) @@ -183,8 +184,8 @@ class NotificationGenerator extends HouseKeeper { Document observationDocument = new Document(observationData) - System.err.println(" ** Made change observation for instance: " + instanceId + " , category: " + categoryUri + " , notes: " + changeNotes + - "\n resulting document: " + observationDocument.getDataAsString()) + //System.err.println(" ** Made change observation for instance: " + instanceId + " , category: " + categoryUri + " , notes: " + changeNotes + + // "\n resulting document: " + observationDocument.getDataAsString()) if (!whelk.createDocument(observationDocument, "NotificationGenerator", "SEK", "none", false)) { log.error("Failed to create ChangeObservation for $instanceId ($categoryUri).") diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 4040173f6b..7881d2669d 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -6,47 +6,54 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import groovy.transform.CompileStatic import groovy.util.logging.Log4j2 as Log - import java.time.ZonedDateTime +import it.sauronsoftware.cron4j.Scheduler @CompileStatic +@Log public abstract class HouseKeeper { public abstract String getName() public abstract String getStatusDescription() + public abstract String getCronSchedule() public abstract void trigger() public ZonedDateTime lastFailAt = null public ZonedDateTime lastRunAt = null + + synchronized void _trigger() { + try { + trigger() + lastRunAt = ZonedDateTime.now() + } catch (Throwable e) { + log.error("Could not handle throwable in Housekeeper TimerTask.", e) + lastFailAt = ZonedDateTime.now() + } + } } @CompileStatic @Log public class WebInterface extends HttpServlet { - private static final long PERIODIC_TRIGGER_MS = 10 * 1000 - private final Timer timer = new Timer("Housekeeper-timer", true) private List houseKeepers = [] + Scheduler cronScheduler = new Scheduler() public void init() { Whelk whelk = Whelk.createLoadedCoreWhelk() houseKeepers = [] houseKeepers.add(new NotificationGenerator(whelk)) + houseKeepers.add(new NotificationSender(whelk)) - for (HouseKeeper hk : houseKeepers) { - timer.scheduleAtFixedRate({ - try { - hk.trigger() - hk.lastRunAt = ZonedDateTime.now() - } catch (Throwable e) { - log.error("Could not handle throwable in Housekeeper TimerTask.", e) - hk.lastFailAt = ZonedDateTime.now() - } - }, PERIODIC_TRIGGER_MS, PERIODIC_TRIGGER_MS) + houseKeepers.each { hk -> + cronScheduler.schedule(hk.getCronSchedule(), { + hk._trigger() + }) } - + cronScheduler.start() } public void destroy() { + cronScheduler.stop() } public void doGet(HttpServletRequest req, HttpServletResponse res) { From 8ae961ac0fbf13cdbbdcb5e82d7d4f85d1969edb Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 2 Nov 2023 13:25:32 +0100 Subject: [PATCH 46/58] Not yet working. --- .../housekeeping/NotificationSender.groovy | 258 ++++++++++++++++++ .../whelk/housekeeping/WebInterface.groovy | 23 +- 2 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy new file mode 100644 index 0000000000..e5f2949cb4 --- /dev/null +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -0,0 +1,258 @@ +package whelk.housekeeping + +import groovy.transform.CompileStatic +import groovy.util.logging.Log4j2 +import org.simplejavamail.api.email.Email +import org.simplejavamail.api.mailer.Mailer +import org.simplejavamail.email.EmailBuilder +import org.simplejavamail.mailer.MailerBuilder +import whelk.Document +import whelk.Whelk +import whelk.util.PropertyLoader + +import java.sql.Array +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.Timestamp +import java.time.Instant +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +@CompileStatic +@Log4j2 +class NotificationSender extends HouseKeeper { + + private final String STATE_KEY = "CXZ notification email sender" + private String status = "OK" + private final Whelk whelk + private final Mailer mailer + private final String senderAddress + + public NotificationSender(Whelk whelk) { + this.whelk = whelk + Properties props = PropertyLoader.loadProperties("secret") + if (props.containsKey("smtpServer") && + props.containsKey("smtpPort") && + props.containsKey("smtpSender") && + props.containsKey("smtpUser") && + props.containsKey("smtpPassword")) + mailer = MailerBuilder + .withSMTPServer( + (String) props.get("smtpServer"), + Integer.parseInt((String)props.get("smtpPort")), + (String) props.get("smtpUser"), + (String) props.get("smtpPassword") + ).buildMailer() + senderAddress = props.get("smtpSender") + } + + @Override + String getName() { + return "Notifications sender" + } + + @Override + String getStatusDescription() { + return status + } + + public String getCronSchedule() { + return "* * * * *" + } + + @Override + void trigger() { + // Build a multi-map of library -> list of settings objects for that library's users + Map> heldByToUserSettings = new HashMap<>(); + { + List allUserSettingStrings = whelk.getStorage().getAllUserData() + for (Map settings : allUserSettingStrings) { + if (!settings["notificationEmail"]) + continue + settings?.requestedNotifications?.each { request -> + if (!request instanceof Map) + return + if (!request["heldBy"]) + return + + String heldBy = request["heldBy"] + if (!heldByToUserSettings.containsKey(heldBy)) + heldByToUserSettings.put(heldBy, []) + heldByToUserSettings[heldBy].add(settings) + } + } + } + + // Determine the time interval of ChangeObservations to consider + Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. + Map sendState = whelk.getStorage().getState(STATE_KEY) + if (sendState && sendState.lastEmailTime) + from = Timestamp.from( ZonedDateTime.parse( (String) sendState.lastEmailTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) + + Timestamp until = null + Map generationState = whelk.getStorage().getState(NotificationGenerator.STATE_KEY) + if (generationState && generationState.lastGenerationTime) + until = Timestamp.from( ZonedDateTime.parse( (String) generationState.lastGenerationTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) + if (!until) + return + + + Connection connection + PreparedStatement statement + ResultSet resultSet + + connection = whelk.getStorage().getOuterConnection() + connection.setAutoCommit(false) + try { + // Fetch all changed IDs within the interval + String sql = "SELECT id, data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND ( created > ? AND created <= ? );" + connection.setAutoCommit(false) + statement = connection.prepareStatement(sql) + statement.setTimestamp(1, from) + statement.setTimestamp(2, until) + System.err.println(" ** Searching for Observations: " + statement) + statement.setFetchSize(512) + resultSet = statement.executeQuery() + + while (resultSet.next()) { + String id = resultSet.getString("id") + System.err.println(" ****** ChangeObservation: " + id + " picked up for emailing!") + } + } catch (Throwable e) { + status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() + throw e + } finally { + connection.close() + Map newState = new HashMap() + newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() + whelk.getStorage().putState(STATE_KEY, newState) + } + + } + + /** + * Generate notifications for an affected bibliographic instance. Beware: fromVersion and untilVersion may not be + * _of this document_ (id), but rather of a document this instance depends on! + */ + + private void generateNotificationsForAffectedInstance(String id, Map heldByToUserSettings, Instant from, Instant until) { + List libraries = whelk.getStorage().getAllLibrariesHolding(id) + for (String library : libraries) { + List users = (List) heldByToUserSettings[library] + if (users) { + for (Map user : users) { + /* + 'user' is now a map looking something like this: + { + "id": "sldknfslkdnsdlkgnsdkjgnb" + "requestedNotifications": [ + { + "heldBy": "https://libris.kb.se/library/Utb1", + "triggers": [ + "https://id.kb.se/changenote/primarytitle" + ] + } + ], + "email": "noreply@kb.se" + }*/ + + List triggered = changeMatchesAnyTrigger( + id, from, + until, user, library) + if (triggered) { + + /* + if (!notificationsByUser.containsKey(user.notificationEmail)) + notificationsByUser.put((String)user.notificationEmail, []) + notificationsByUser[user.notificationEmail].add(generateEmailBody(id, triggered)) + */ + System.err.println("DO IT, MAKE A NOTIFICATION RECORD FOR INSTANCE " + id + " : " + triggered) + } + } + } + } + } + + /** + * 'from' and 'until' are the instants of writing for the changed record (the previous and current versions) + * 'user' is the user-data map for a user (which includes their selection of triggers). + * 'library' is a library ("sigel") holding the instance in question. + * + * This function answers the question: Has 'user' requested to be notified of the occurred changes between + * 'from' and 'until' for instances held by 'heldBy'? + * + * Returns the URIs of all triggered rules/triggers. + */ + private List changeMatchesAnyTrigger(String instanceId, Instant from, Instant until, Map user, String heldBy) { + + List triggeredTriggers = [] + + user.requestedNotifications.each { request -> + + // This stuff (the request) comes from a user, so we must be super paranoid about it being correctly formed. + + if (! request instanceof Map) + return + + if (! request["heldBy"] instanceof String) + return + + if (request["heldBy"] != heldBy) + return + + if (! request["triggers"] instanceof List) + return + + for (Object triggerObject : request["triggers"]) { + if (! triggerObject instanceof String) + return + String triggerUri = (String) triggerObject + /*if (triggerIsTriggered(instanceId, from, until, triggerUri)) + triggeredTriggers.add(triggerUri)*/ + } + } + + return triggeredTriggers + } + + private void sendEmail(String sender, String recipient, String subject, String body) { + if (mailer) { + Email email = EmailBuilder.startingBlank() + .to(recipient) + .withSubject(subject) + .from(sender) + .withPlainText(body) + .buildEmail() + + log.info("Sending notification (cxz) email to " + recipient) + mailer.sendMail(email) + } else { + log.info("Should now have sent notification (cxz) email to " + recipient + " but SMTP not configured.") + } + } + + private String generateEmailBody(String changedInstanceId, List triggeredCategories) { + Document changed = whelk.getStorage().load(changedInstanceId) + String mainTitle = Document._get(["@graph", 1, "hasTitle", 0, "mainTitle"], changed.data) + if (mainTitle == null) + mainTitle = "Ingen huvudtitel" + + String changeCategories = "" + for (String categoryUri : triggeredCategories) { + Document categoryDoc = whelk.getStorage().loadDocumentByMainId(categoryUri) + String swedishLabel = Document._get(["@graph", 1, "prefLabelByLand", "sv"], categoryDoc.data) + if (swedishLabel) { + changeCategories += swedishLabel + " " + } else { + changeCategories += categoryUri + " " + } + } + + return "Instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(changedInstanceId) + + "(" + mainTitle + ")\n\t" + + "Ändring har skett med avseende på: " + changeCategories + "\n\n" + } +} diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 7881d2669d..392eccee86 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -34,20 +34,22 @@ public abstract class HouseKeeper { @CompileStatic @Log public class WebInterface extends HttpServlet { - private List houseKeepers = [] + private Map houseKeepersById = [:] Scheduler cronScheduler = new Scheduler() public void init() { Whelk whelk = Whelk.createLoadedCoreWhelk() - houseKeepers = [] - houseKeepers.add(new NotificationGenerator(whelk)) - houseKeepers.add(new NotificationSender(whelk)) + List houseKeepers = [ + new NotificationGenerator(whelk), + new NotificationSender(whelk) + ] houseKeepers.each { hk -> - cronScheduler.schedule(hk.getCronSchedule(), { + String id = cronScheduler.schedule(hk.getCronSchedule(), { hk._trigger() }) + houseKeepersById.put(id, hk) } cronScheduler.start() } @@ -58,9 +60,10 @@ public class WebInterface extends HttpServlet { public void doGet(HttpServletRequest req, HttpServletResponse res) { StringBuilder sb = new StringBuilder() - sb.append("Active housekeepers: " + houseKeepers.size() + "\n") + sb.append("Active housekeepers: " + houseKeepersById.size() + "\n") sb.append("--------------\n") - for (HouseKeeper hk : houseKeepers) { + for (String key : houseKeepersById.keySet()) { + HouseKeeper hk = houseKeepersById[key] sb.append(hk.getName() + "\n") if (hk.lastRunAt) sb.append("last run at: " + hk.lastRunAt + "\n") @@ -72,10 +75,16 @@ public class WebInterface extends HttpServlet { sb.append("no failures\n") sb.append("status:\n") sb.append(hk.statusDescription+"\n") + sb.append("To force immediate execution, POST to:\n" + req.getRequestURL() + key + "\n") sb.append("--------------\n") } res.setStatus(HttpServletResponse.SC_OK) res.setContentType("text/plain") res.getOutputStream().print(sb.toString()) } + + public void doPost(HttpServletRequest req, HttpServletResponse res) { + String key = req.getRequestURI().split("/").last() + houseKeepersById[key]._trigger() + } } From 26859229ff357c7c8052e52138d1d2b412fd9a21 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 2 Nov 2023 14:06:35 +0100 Subject: [PATCH 47/58] Fix a time overlap bug. --- .../main/groovy/whelk/housekeeping/NotificationSender.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index e5f2949cb4..73f5909d71 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -96,7 +96,9 @@ class NotificationSender extends HouseKeeper { Map generationState = whelk.getStorage().getState(NotificationGenerator.STATE_KEY) if (generationState && generationState.lastGenerationTime) until = Timestamp.from( ZonedDateTime.parse( (String) generationState.lastGenerationTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) - if (!until) + if (until == (Object) null) // Groovy... + return + if (until.toInstant().isBefore(from.toInstant())) return From 5f21e68a3df99cd4c015398931ccf5f2a24bb134 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 3 Nov 2023 10:25:07 +0100 Subject: [PATCH 48/58] Almost working email cxz --- .../housekeeping/NotificationGenerator.groovy | 3 +- .../housekeeping/NotificationSender.groovy | 124 +++++++----------- .../whelk/housekeeping/WebInterface.groovy | 2 +- 3 files changed, 48 insertions(+), 81 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 04b92cc6bd..5cda5c8d44 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -148,7 +148,7 @@ class NotificationGenerator extends HouseKeeper { contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] ) - makeChangeObservation(instanceId, changeNotes, "https://id.kb.se/changenote/primarycontribution", (Map) contrBefore, (Map) contrAfter) + makeChangeObservation(instanceId, changeNotes, "https://id.kb.se/changecategory/primarycontribution", (Map) contrBefore, (Map) contrAfter) } } } @@ -174,6 +174,7 @@ class NotificationGenerator extends HouseKeeper { "about" : ["@id" : Document.BASE_URI.toString() + instanceId], "representationBefore" : oldValue, "representationAfter" : newValue, + "category" : categoryUri, ] ]] diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 73f5909d71..4d511aa00d 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -21,6 +21,8 @@ import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit +import static whelk.util.Jackson.mapper + @CompileStatic @Log4j2 class NotificationSender extends HouseKeeper { @@ -110,7 +112,8 @@ class NotificationSender extends HouseKeeper { connection.setAutoCommit(false) try { // Fetch all changed IDs within the interval - String sql = "SELECT id, data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND ( created > ? AND created <= ? );" + //String sql = "SELECT id, data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND ( created > ? AND created <= ? );" + String sql = "SELECT data#>>'{@graph,1,about,@id}' as instanceUri, ARRAY_AGG(data::text) as data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND ( created > ? AND created <= ? ) GROUP BY data#>>'{@graph,1,about,@id}';" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -120,8 +123,17 @@ class NotificationSender extends HouseKeeper { resultSet = statement.executeQuery() while (resultSet.next()) { - String id = resultSet.getString("id") - System.err.println(" ****** ChangeObservation: " + id + " picked up for emailing!") + String instanceUri = resultSet.getString("instanceUri") + Array changeObservationsArray = resultSet.getArray("data") + // Groovy.. + List changeObservationsForInstance = [] + for (Object o : changeObservationsArray.getArray()) { + changeObservationsForInstance.add(o) + } + + System.err.println("About to email for instance: " + instanceUri) + + sendFor(instanceUri, heldByToUserSettings, changeObservationsForInstance) } } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() @@ -135,13 +147,10 @@ class NotificationSender extends HouseKeeper { } - /** - * Generate notifications for an affected bibliographic instance. Beware: fromVersion and untilVersion may not be - * _of this document_ (id), but rather of a document this instance depends on! - */ + private void sendFor(String instanceUri, Map> heldByToUserSettings, List changeObservationsForInstance) { + String instanceId = whelk.getStorage().getSystemIdByIri(instanceUri) + List libraries = whelk.getStorage().getAllLibrariesHolding(instanceId) - private void generateNotificationsForAffectedInstance(String id, Map heldByToUserSettings, Instant from, Instant until) { - List libraries = whelk.getStorage().getAllLibrariesHolding(id) for (String library : libraries) { List users = (List) heldByToUserSettings[library] if (users) { @@ -154,70 +163,45 @@ class NotificationSender extends HouseKeeper { { "heldBy": "https://libris.kb.se/library/Utb1", "triggers": [ - "https://id.kb.se/changenote/primarytitle" + "https://id.kb.se/changecategory/primarycontribution" ] } ], - "email": "noreply@kb.se" + "notificationEmail": "noreply@kb.se" }*/ - List triggered = changeMatchesAnyTrigger( - id, from, - until, user, library) - if (triggered) { + List triggeredCategories = [] + + user?.requestedNotifications?.each { Map request -> + request?.triggers?.each { String trigger -> + if (matches(trigger, changeObservationsForInstance)) { + triggeredCategories.add(trigger) + } + } + } + + if (!triggeredCategories.isEmpty() && user.notificationEmail && user.notificationEmail instanceof String) { + String body = generateEmailBody(instanceId, triggeredCategories) + sendEmail(senderAddress, (String) user.notificationEmail, "CXZ", body) - /* - if (!notificationsByUser.containsKey(user.notificationEmail)) - notificationsByUser.put((String)user.notificationEmail, []) - notificationsByUser[user.notificationEmail].add(generateEmailBody(id, triggered)) - */ - System.err.println("DO IT, MAKE A NOTIFICATION RECORD FOR INSTANCE " + id + " : " + triggered) + System.err.println("Now send email to " + user.notificationEmail + "\n\t" + body) } } } } - } - - /** - * 'from' and 'until' are the instants of writing for the changed record (the previous and current versions) - * 'user' is the user-data map for a user (which includes their selection of triggers). - * 'library' is a library ("sigel") holding the instance in question. - * - * This function answers the question: Has 'user' requested to be notified of the occurred changes between - * 'from' and 'until' for instances held by 'heldBy'? - * - * Returns the URIs of all triggered rules/triggers. - */ - private List changeMatchesAnyTrigger(String instanceId, Instant from, Instant until, Map user, String heldBy) { - - List triggeredTriggers = [] - - user.requestedNotifications.each { request -> - // This stuff (the request) comes from a user, so we must be super paranoid about it being correctly formed. - - if (! request instanceof Map) - return - - if (! request["heldBy"] instanceof String) - return - - if (request["heldBy"] != heldBy) - return - - if (! request["triggers"] instanceof List) - return + } - for (Object triggerObject : request["triggers"]) { - if (! triggerObject instanceof String) - return - String triggerUri = (String) triggerObject - /*if (triggerIsTriggered(instanceId, from, until, triggerUri)) - triggeredTriggers.add(triggerUri)*/ - } + private boolean matches(String trigger, List changeObservationsForInstance) { + for (Object obj : changeObservationsForInstance) { + Map changeObservationMap = mapper.readValue( (String) obj, Map ) + List graphList = changeObservationMap["@graph"] + Map mainEntity = graphList?[1] + String category = mainEntity?.category + if (category && category == trigger) + return true } - - return triggeredTriggers + return false } private void sendEmail(String sender, String recipient, String subject, String body) { @@ -237,24 +221,6 @@ class NotificationSender extends HouseKeeper { } private String generateEmailBody(String changedInstanceId, List triggeredCategories) { - Document changed = whelk.getStorage().load(changedInstanceId) - String mainTitle = Document._get(["@graph", 1, "hasTitle", 0, "mainTitle"], changed.data) - if (mainTitle == null) - mainTitle = "Ingen huvudtitel" - - String changeCategories = "" - for (String categoryUri : triggeredCategories) { - Document categoryDoc = whelk.getStorage().loadDocumentByMainId(categoryUri) - String swedishLabel = Document._get(["@graph", 1, "prefLabelByLand", "sv"], categoryDoc.data) - if (swedishLabel) { - changeCategories += swedishLabel + " " - } else { - changeCategories += categoryUri + " " - } - } - - return "Instansbeskrivning har ändrats\n\tInstans: " + Document.BASE_URI.resolve(changedInstanceId) + - "(" + mainTitle + ")\n\t" + - "Ändring har skett med avseende på: " + changeCategories + "\n\n" + return "Ändring av " + changedInstanceId + " kategorier: " + triggeredCategories } } diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 392eccee86..83218a2cb6 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -38,7 +38,7 @@ public class WebInterface extends HttpServlet { Scheduler cronScheduler = new Scheduler() public void init() { - Whelk whelk = Whelk.createLoadedCoreWhelk() + Whelk whelk = Whelk.createLoadedSearchWhelk() List houseKeepers = [ new NotificationGenerator(whelk), From c1dd4fc69d320fbfabcbebb9365dd148401f57ec Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 3 Nov 2023 10:43:26 +0100 Subject: [PATCH 49/58] Progress --- .../housekeeping/NotificationSender.groovy | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 4d511aa00d..69be5d9ee7 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -91,33 +91,22 @@ class NotificationSender extends HouseKeeper { // Determine the time interval of ChangeObservations to consider Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. Map sendState = whelk.getStorage().getState(STATE_KEY) - if (sendState && sendState.lastEmailTime) - from = Timestamp.from( ZonedDateTime.parse( (String) sendState.lastEmailTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) - - Timestamp until = null - Map generationState = whelk.getStorage().getState(NotificationGenerator.STATE_KEY) - if (generationState && generationState.lastGenerationTime) - until = Timestamp.from( ZonedDateTime.parse( (String) generationState.lastGenerationTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) - if (until == (Object) null) // Groovy... - return - if (until.toInstant().isBefore(from.toInstant())) - return - + if (sendState && sendState.notifiedChangesUpTo) + from = Timestamp.from( ZonedDateTime.parse( (String) sendState.notifiedChangesUpTo, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) Connection connection PreparedStatement statement ResultSet resultSet + Instant notifiedChangesUpTo = Instant.EPOCH + connection = whelk.getStorage().getOuterConnection() connection.setAutoCommit(false) try { - // Fetch all changed IDs within the interval - //String sql = "SELECT id, data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND ( created > ? AND created <= ? );" - String sql = "SELECT data#>>'{@graph,1,about,@id}' as instanceUri, ARRAY_AGG(data::text) as data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND ( created > ? AND created <= ? ) GROUP BY data#>>'{@graph,1,about,@id}';" + String sql = "SELECT MAX(created) as lastChange, data#>>'{@graph,1,about,@id}' as instanceUri, ARRAY_AGG(data::text) as data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND created > ? GROUP BY data#>>'{@graph,1,about,@id}';" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) - statement.setTimestamp(2, until) System.err.println(" ** Searching for Observations: " + statement) statement.setFetchSize(512) resultSet = statement.executeQuery() @@ -134,6 +123,10 @@ class NotificationSender extends HouseKeeper { System.err.println("About to email for instance: " + instanceUri) sendFor(instanceUri, heldByToUserSettings, changeObservationsForInstance) + + Instant lastChangeObservationForInstance = resultSet.getTimestamp("lastChange").toInstant() + if (lastChangeObservationForInstance.isAfter(notifiedChangesUpTo)) + notifiedChangesUpTo = lastChangeObservationForInstance } } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() @@ -141,7 +134,7 @@ class NotificationSender extends HouseKeeper { } finally { connection.close() Map newState = new HashMap() - newState.lastEmailTime = until.toInstant().atOffset(ZoneOffset.UTC).toString() + newState.notifiedChangesUpTo = notifiedChangesUpTo.atOffset(ZoneOffset.UTC).toString() whelk.getStorage().putState(STATE_KEY, newState) } From df71a7af3ce5ec12c9618aa7d583009b6f807d1b Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 3 Nov 2023 11:02:07 +0100 Subject: [PATCH 50/58] Fix a tiemstamp reset to EPOCH bug. --- .../whelk/housekeeping/NotificationSender.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 69be5d9ee7..0339edcb30 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -19,7 +19,6 @@ import java.time.Instant import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter -import java.time.temporal.ChronoUnit import static whelk.util.Jackson.mapper @@ -89,7 +88,7 @@ class NotificationSender extends HouseKeeper { } // Determine the time interval of ChangeObservations to consider - Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. + Timestamp from = Timestamp.from(Instant.EPOCH) Map sendState = whelk.getStorage().getState(STATE_KEY) if (sendState && sendState.notifiedChangesUpTo) from = Timestamp.from( ZonedDateTime.parse( (String) sendState.notifiedChangesUpTo, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) @@ -133,9 +132,11 @@ class NotificationSender extends HouseKeeper { throw e } finally { connection.close() - Map newState = new HashMap() - newState.notifiedChangesUpTo = notifiedChangesUpTo.atOffset(ZoneOffset.UTC).toString() - whelk.getStorage().putState(STATE_KEY, newState) + if (notifiedChangesUpTo.isAfter(from.toInstant())) { + Map newState = new HashMap() + newState.notifiedChangesUpTo = notifiedChangesUpTo.atOffset(ZoneOffset.UTC).toString() + whelk.getStorage().putState(STATE_KEY, newState) + } } } From 0669e7731a1313d0c1cfd870f26b3bc896c77e00 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Fri, 3 Nov 2023 11:13:56 +0100 Subject: [PATCH 51/58] Don't scan too far back. --- .../main/groovy/whelk/housekeeping/NotificationSender.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 0339edcb30..ab8fa7c98c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -19,6 +19,7 @@ import java.time.Instant import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit import static whelk.util.Jackson.mapper @@ -88,7 +89,7 @@ class NotificationSender extends HouseKeeper { } // Determine the time interval of ChangeObservations to consider - Timestamp from = Timestamp.from(Instant.EPOCH) + Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. Map sendState = whelk.getStorage().getState(STATE_KEY) if (sendState && sendState.notifiedChangesUpTo) from = Timestamp.from( ZonedDateTime.parse( (String) sendState.notifiedChangesUpTo, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) @@ -97,7 +98,7 @@ class NotificationSender extends HouseKeeper { PreparedStatement statement ResultSet resultSet - Instant notifiedChangesUpTo = Instant.EPOCH + Instant notifiedChangesUpTo = from.toInstant() connection = whelk.getStorage().getOuterConnection() connection.setAutoCommit(false) From 9c1f709e6ba42be01153ccbf114578c76e1cf80d Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 6 Nov 2023 10:42:54 +0100 Subject: [PATCH 52/58] Add a threshold value, so that a change cant result in too many observation records. --- .../housekeeping/NotificationGenerator.groovy | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 5cda5c8d44..b69af3481b 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -25,6 +25,7 @@ import static whelk.util.Jackson.mapper class NotificationGenerator extends HouseKeeper { public static final String STATE_KEY = "CXZ notification generator" + private static final int MAX_OBSERVATIONS_PER_CHANGE = 100 private String status = "OK" private final Whelk whelk @@ -101,22 +102,38 @@ class NotificationGenerator extends HouseKeeper { * relevant notification-records */ private void generateNotificationsForChangedID(String id, List changeNotes, Instant from, Instant until, Set affectedInstanceIDs) { + + List resultingChangeObservations = [] + List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! dependers.each { String dependerID = it[0] String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { - // If we've not already sent a notification for this instance! + // If we've not already made an observation for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - generateNotificationsForAffectedInstance(dependerID, changeNotes, from, until) + resultingChangeObservations.addAll( generateNotificationsForAffectedInstance(dependerID, changeNotes, from, until) ) + if (resultingChangeObservations.size() > MAX_OBSERVATIONS_PER_CHANGE) { + log.warn("Discarding ChangeObservations for instances related to $id, which was changed. Observations would be too many.") + return + } } } } + + for (Document observation : resultingChangeObservations) { + System.err.println(" ** Made change observation :${observation?.data}") + + if (!whelk.createDocument(observation, "NotificationGenerator", "SEK", "none", false)) { + log.error("Failed to create ChangeObservation for ${observation?.data[1]["about"]["@id"]} (${observation?.data[1]["category"]["@id"]}).") + } + } } - private void generateNotificationsForAffectedInstance(String instanceId, List changeNotes, Instant before, Instant after) { + private List generateNotificationsForAffectedInstance(String instanceId, List changeNotes, Instant before, Instant after) { + List generatedObservations = [] List propertiesToEmbellish = [ "mainEntity", "instanceOf", @@ -148,16 +165,21 @@ class NotificationGenerator extends HouseKeeper { contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] ) - makeChangeObservation(instanceId, changeNotes, "https://id.kb.se/changecategory/primarycontribution", (Map) contrBefore, (Map) contrAfter) + generatedObservations.add( + makeChangeObservation(instanceId, changeNotes, + "https://id.kb.se/changecategory/primarycontribution", + (Map) contrBefore, (Map) contrAfter)) } } } } } } + + return generatedObservations } - private void makeChangeObservation(String instanceId, List changeNotes, String categoryUri, Map oldValue, Map newValue) { + private Document makeChangeObservation(String instanceId, List changeNotes, String categoryUri, Map oldValue, Map newValue) { String newId = IdGenerator.generate() String metadataUri = Document.BASE_URI.toString() + newId String mainEntityUri = metadataUri+"#it" @@ -174,7 +196,7 @@ class NotificationGenerator extends HouseKeeper { "about" : ["@id" : Document.BASE_URI.toString() + instanceId], "representationBefore" : oldValue, "representationAfter" : newValue, - "category" : categoryUri, + "category" : ["@id" : categoryUri], ] ]] @@ -183,14 +205,7 @@ class NotificationGenerator extends HouseKeeper { observationData["@graph"][1]["comment"] = comments } - Document observationDocument = new Document(observationData) - - //System.err.println(" ** Made change observation for instance: " + instanceId + " , category: " + categoryUri + " , notes: " + changeNotes + - // "\n resulting document: " + observationDocument.getDataAsString()) - - if (!whelk.createDocument(observationDocument, "NotificationGenerator", "SEK", "none", false)) { - log.error("Failed to create ChangeObservation for $instanceId ($categoryUri).") - } + return new Document(observationData) } private List extractComments(List changeNotes) { From af1f61a93bd0256a6b2cacfb6804915824fac31d Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 6 Nov 2023 11:05:10 +0100 Subject: [PATCH 53/58] Clean up logging. --- .../groovy/whelk/housekeeping/NotificationGenerator.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index b69af3481b..2ca26e453c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -124,10 +124,10 @@ class NotificationGenerator extends HouseKeeper { } for (Document observation : resultingChangeObservations) { - System.err.println(" ** Made change observation :${observation?.data}") + //System.err.println(" ** Made change observation:\n${observation.getDataAsString()}") if (!whelk.createDocument(observation, "NotificationGenerator", "SEK", "none", false)) { - log.error("Failed to create ChangeObservation for ${observation?.data[1]["about"]["@id"]} (${observation?.data[1]["category"]["@id"]}).") + log.error("Failed to create ChangeObservation:\n${observation.getDataAsString()}") } } } From 61529449cb7614bcb6cd2cdde5eb7c9ea815957d Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Mon, 6 Nov 2023 14:37:40 +0100 Subject: [PATCH 54/58] Progress towards proper message bodies. --- .../housekeeping/NotificationGenerator.groovy | 8 ++-- .../housekeeping/NotificationSender.groovy | 38 +++++++++++++------ .../whelk/housekeeping/WebInterface.groovy | 12 +++--- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 2ca26e453c..d533184e6c 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -83,7 +83,7 @@ class NotificationGenerator extends HouseKeeper { changeNotes.add(o) } - generateNotificationsForChangedID(id, changeNotes, from.toInstant(), + generateObservationsForChangedID(id, changeNotes, from.toInstant(), until.toInstant(), affectedInstanceIDs) } } catch (Throwable e) { @@ -101,7 +101,7 @@ class NotificationGenerator extends HouseKeeper { * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) * relevant notification-records */ - private void generateNotificationsForChangedID(String id, List changeNotes, Instant from, Instant until, Set affectedInstanceIDs) { + private void generateObservationsForChangedID(String id, List changeNotes, Instant from, Instant until, Set affectedInstanceIDs) { List resultingChangeObservations = [] @@ -114,7 +114,7 @@ class NotificationGenerator extends HouseKeeper { // If we've not already made an observation for this instance! if (!affectedInstanceIDs.contains(dependerID)) { affectedInstanceIDs.add(dependerID) - resultingChangeObservations.addAll( generateNotificationsForAffectedInstance(dependerID, changeNotes, from, until) ) + resultingChangeObservations.addAll( generateObservationsForAffectedInstance(dependerID, changeNotes, from, until) ) if (resultingChangeObservations.size() > MAX_OBSERVATIONS_PER_CHANGE) { log.warn("Discarding ChangeObservations for instances related to $id, which was changed. Observations would be too many.") return @@ -132,7 +132,7 @@ class NotificationGenerator extends HouseKeeper { } } - private List generateNotificationsForAffectedInstance(String instanceId, List changeNotes, Instant before, Instant after) { + private List generateObservationsForAffectedInstance(String instanceId, List changeNotes, Instant before, Instant after) { List generatedObservations = [] List propertiesToEmbellish = [ "mainEntity", diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index ab8fa7c98c..e65e27875e 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -7,6 +7,7 @@ import org.simplejavamail.api.mailer.Mailer import org.simplejavamail.email.EmailBuilder import org.simplejavamail.mailer.MailerBuilder import whelk.Document +import whelk.JsonLd import whelk.Whelk import whelk.util.PropertyLoader @@ -165,18 +166,19 @@ class NotificationSender extends HouseKeeper { "notificationEmail": "noreply@kb.se" }*/ - List triggeredCategories = [] + List matchedObservations = [] user?.requestedNotifications?.each { Map request -> request?.triggers?.each { String trigger -> - if (matches(trigger, changeObservationsForInstance)) { - triggeredCategories.add(trigger) + Map triggeredObservation = matches(trigger, changeObservationsForInstance) + if (triggeredObservation != null) { + matchedObservations.add(triggeredObservation) } } } - if (!triggeredCategories.isEmpty() && user.notificationEmail && user.notificationEmail instanceof String) { - String body = generateEmailBody(instanceId, triggeredCategories) + if (!matchedObservations.isEmpty() && user.notificationEmail && user.notificationEmail instanceof String) { + String body = generateEmailBody(instanceId, matchedObservations) sendEmail(senderAddress, (String) user.notificationEmail, "CXZ", body) System.err.println("Now send email to " + user.notificationEmail + "\n\t" + body) @@ -187,16 +189,16 @@ class NotificationSender extends HouseKeeper { } - private boolean matches(String trigger, List changeObservationsForInstance) { + private Map matches(String trigger, List changeObservationsForInstance) { for (Object obj : changeObservationsForInstance) { Map changeObservationMap = mapper.readValue( (String) obj, Map ) List graphList = changeObservationMap["@graph"] Map mainEntity = graphList?[1] - String category = mainEntity?.category + String category = mainEntity?.category["@id"] if (category && category == trigger) - return true + return changeObservationMap } - return false + return null } private void sendEmail(String sender, String recipient, String subject, String body) { @@ -215,7 +217,21 @@ class NotificationSender extends HouseKeeper { } } - private String generateEmailBody(String changedInstanceId, List triggeredCategories) { - return "Ändring av " + changedInstanceId + " kategorier: " + triggeredCategories + private String generateEmailBody(String changedInstanceId, List triggeredObservations) { + StringBuilder sb = new StringBuilder() + for (Map observation : triggeredObservations) { + String observationUri = Document._get(["@graph", 1, "@id"], observation) + if (!observationUri) + continue + + String observationId = whelk.getStorage().getSystemIdByIri(observationUri) + Document embellishedObservation = whelk.loadEmbellished(observationId) + //Map mainEntity = (Map) Document._get(["@graph", 1], embellishedObservation.data) + Map framed = JsonLd.frame(observationUri, embellishedObservation.data) + Map filtered = whelk.getJsonld().applyLensAsMapByLang(framed, ["sv", "en"] as Set, [], ["cards", "chips"]) + sb.append("For observation " + observationId + ":\n\t" + filtered) + } + + return sb.toString() } } diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy index 83218a2cb6..11736e513e 100755 --- a/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/WebInterface.groovy @@ -66,15 +66,17 @@ public class WebInterface extends HttpServlet { HouseKeeper hk = houseKeepersById[key] sb.append(hk.getName() + "\n") if (hk.lastRunAt) - sb.append("last run at: " + hk.lastRunAt + "\n") + sb.append("Last run at: " + hk.lastRunAt + "\n") else - sb.append("has never run\n") + sb.append("Has never run\n") if (hk.lastFailAt) - sb.append("last failed at: " + hk.lastFailAt + "\n") + sb.append("Last failed at: " + hk.lastFailAt + "\n") else - sb.append("no failures\n") - sb.append("status:\n") + sb.append("No failures\n") + sb.append("Status:\n") sb.append(hk.statusDescription+"\n") + sb.append("Execution schedule:\n") + sb.append(hk.cronSchedule+"\n") sb.append("To force immediate execution, POST to:\n" + req.getRequestURL() + key + "\n") sb.append("--------------\n") } From 0c1f5d4d9780fbeb1f05bd15af3a18e202140ed2 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 8 Nov 2023 12:00:01 +0100 Subject: [PATCH 55/58] Fixes for various problems. --- .../housekeeping/NotificationGenerator.groovy | 31 +++++++++++++------ .../housekeeping/NotificationSender.groovy | 13 +++++--- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index d533184e6c..62b7cae76d 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -47,11 +47,12 @@ class NotificationGenerator extends HouseKeeper { public void trigger() { // Determine the time interval of changes for which to generate notifications. - Timestamp from = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)) // Default to last 24h if first time. + Instant now = Instant.now() + Timestamp from = Timestamp.from(now) // First run? Default to now (=do nothing but set the timestamp for next run) Map state = whelk.getStorage().getState(STATE_KEY) if (state && state.lastGenerationTime) from = Timestamp.from( ZonedDateTime.parse( (String) state.lastGenerationTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant() ) - Timestamp until = Timestamp.from(Instant.now()) + Timestamp until = Timestamp.from(now) Connection connection PreparedStatement statement @@ -149,6 +150,9 @@ class NotificationGenerator extends HouseKeeper { Document instanceAfterChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(after)) historicEmbellish(instanceAfterChange, propertiesToEmbellish, after) Document instanceBeforeChange = whelk.getStorage().loadAsOf(instanceId, Timestamp.from(before)) + if (instanceBeforeChange == null) { // This instance is new, and did not exist at 'before'. + return generatedObservations + } historicEmbellish(instanceBeforeChange, propertiesToEmbellish, before); // Check for primary contribution changes @@ -159,16 +163,16 @@ class NotificationGenerator extends HouseKeeper { for (Object contrBefore : contributionsBefore) { for (Object contrAfter : contributionsAfter) { if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution")) { - if (contributionsBefore["agent"] != null && contributionsAfter["agent"] != null) { + if (contrBefore["agent"] != null && contrAfter["agent"] != null) { if ( - contributionsBefore["agent"]["familyName"] != contributionsAfter["agent"]["familyName"] || - contributionsBefore["agent"]["givenName"] != contributionsAfter["agent"]["givenName"] || - contributionsBefore["agent"]["lifeSpan"] != contributionsAfter["agent"]["lifeSpan"] + contrBefore["agent"]["familyName"] != contrAfter["agent"]["familyName"] || + contrBefore["agent"]["givenName"] != contrAfter["agent"]["givenName"] || + contrBefore["agent"]["lifeSpan"] != contrAfter["agent"]["lifeSpan"] ) generatedObservations.add( makeChangeObservation(instanceId, changeNotes, "https://id.kb.se/changecategory/primarycontribution", - (Map) contrBefore, (Map) contrAfter)) + (Map) contrBefore["agent"], (Map) contrAfter["agent"])) } } } @@ -184,6 +188,12 @@ class NotificationGenerator extends HouseKeeper { String metadataUri = Document.BASE_URI.toString() + newId String mainEntityUri = metadataUri+"#it" + // If the @id is left, the object is considered a link, and the actual data (which we want) is removed when storing this as a record. + Map oldValueEmbedded = new HashMap(oldValue) + oldValueEmbedded.remove("@id") + Map newValueEmbedded = new HashMap(newValue) + newValueEmbedded.remove("@id") + Map observationData = [ "@graph":[ [ "@id" : metadataUri, @@ -194,15 +204,16 @@ class NotificationGenerator extends HouseKeeper { "@id" : mainEntityUri, "@type" : "ChangeObservation", "about" : ["@id" : Document.BASE_URI.toString() + instanceId], - "representationBefore" : oldValue, - "representationAfter" : newValue, + "representationBefore" : oldValueEmbedded, + "representationAfter" : newValueEmbedded, "category" : ["@id" : categoryUri], ] ]] List comments = extractComments(changeNotes) if (comments) { - observationData["@graph"][1]["comment"] = comments + Map mainEntity = (Map) observationData["@graph"][1] + mainEntity.put("comment", comments) } return new Document(observationData) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index e65e27875e..3863558291 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -121,8 +121,6 @@ class NotificationSender extends HouseKeeper { changeObservationsForInstance.add(o) } - System.err.println("About to email for instance: " + instanceUri) - sendFor(instanceUri, heldByToUserSettings, changeObservationsForInstance) Instant lastChangeObservationForInstance = resultSet.getTimestamp("lastChange").toInstant() @@ -226,10 +224,15 @@ class NotificationSender extends HouseKeeper { String observationId = whelk.getStorage().getSystemIdByIri(observationUri) Document embellishedObservation = whelk.loadEmbellished(observationId) - //Map mainEntity = (Map) Document._get(["@graph", 1], embellishedObservation.data) Map framed = JsonLd.frame(observationUri, embellishedObservation.data) - Map filtered = whelk.getJsonld().applyLensAsMapByLang(framed, ["sv", "en"] as Set, [], ["cards", "chips"]) - sb.append("For observation " + observationId + ":\n\t" + filtered) + + //Document temp = new Document(framed) + //System.err.println("Framed emb observation:\n\t"+temp.getDataAsString()+"\n\n") + + Map category = whelk.getJsonld().applyLensAsMapByLang( (Map) framed["category"], ["sv"] as Set, [], ["chips"]) + Map before = whelk.getJsonld().applyLensAsMapByLang( (Map) framed["representationBefore"], ["sv"] as Set, [], ["chips"]) + Map after = whelk.getJsonld().applyLensAsMapByLang( (Map) framed["representationAfter"], ["sv"] as Set, [], ["chips"]) + sb.append("For observation " + observationId + ":\n\t" + category + "\n\t" + before + "\n\t" + after) } return sb.toString() From 4453c08f0862f4b95160a8fbf48e1926f3d84e28 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Wed, 8 Nov 2023 14:14:14 +0100 Subject: [PATCH 56/58] More proper email formatting --- .../housekeeping/NotificationGenerator.groovy | 2 +- .../housekeeping/NotificationSender.groovy | 28 +++++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 62b7cae76d..0a0afcb86a 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -25,7 +25,7 @@ import static whelk.util.Jackson.mapper class NotificationGenerator extends HouseKeeper { public static final String STATE_KEY = "CXZ notification generator" - private static final int MAX_OBSERVATIONS_PER_CHANGE = 100 + private static final int MAX_OBSERVATIONS_PER_CHANGE = 20 private String status = "OK" private final Whelk whelk diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 3863558291..2f5142a184 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -108,7 +108,7 @@ class NotificationSender extends HouseKeeper { connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) - System.err.println(" ** Searching for Observations: " + statement) + //System.err.println(" ** Searching for Observations: " + statement) statement.setFetchSize(512) resultSet = statement.executeQuery() @@ -211,12 +211,22 @@ class NotificationSender extends HouseKeeper { log.info("Sending notification (cxz) email to " + recipient) mailer.sendMail(email) } else { - log.info("Should now have sent notification (cxz) email to " + recipient + " but SMTP not configured.") + log.info("Should now have sent notification (cxz) email to " + recipient + " but SMTP is not configured.") } } private String generateEmailBody(String changedInstanceId, List triggeredObservations) { + + Document current = whelk.getStorage().load(changedInstanceId) + String mainTitle = Document._get(["@graph", 1, "hasTitle", 0, "mainTitle"], current.data) + StringBuilder sb = new StringBuilder() + sb.append("Ändringar har skett i instans: " + Document.BASE_URI.resolve(changedInstanceId).toString()) + if (mainTitle) + sb.append(" (" + mainTitle + ")\n") + else + sb.append("\n") + for (Map observation : triggeredObservations) { String observationUri = Document._get(["@graph", 1, "@id"], observation) if (!observationUri) @@ -226,13 +236,19 @@ class NotificationSender extends HouseKeeper { Document embellishedObservation = whelk.loadEmbellished(observationId) Map framed = JsonLd.frame(observationUri, embellishedObservation.data) - //Document temp = new Document(framed) - //System.err.println("Framed emb observation:\n\t"+temp.getDataAsString()+"\n\n") - Map category = whelk.getJsonld().applyLensAsMapByLang( (Map) framed["category"], ["sv"] as Set, [], ["chips"]) Map before = whelk.getJsonld().applyLensAsMapByLang( (Map) framed["representationBefore"], ["sv"] as Set, [], ["chips"]) Map after = whelk.getJsonld().applyLensAsMapByLang( (Map) framed["representationAfter"], ["sv"] as Set, [], ["chips"]) - sb.append("For observation " + observationId + ":\n\t" + category + "\n\t" + before + "\n\t" + after) + sb.append("\tÄndring avser kategorin: "+ category["sv"]) + sb.append("\n\t\tInnan ändring: " + before["sv"]) + sb.append("\n\t\tEfter ändring: " + after["sv"]) + + if (observation["comment"]) { + sb.append("\n\t\tTillhörande kommentarer:") + for (String comment : observation["comment"]) + sb.append("\n\t\t\t"+comment) + } + sb.append("\n\n") } return sb.toString() From 86ea65d431151141e857d0c65995853dcf07d0da Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 9 Nov 2023 13:52:22 +0100 Subject: [PATCH 57/58] CXZ Comments working too. --- .../housekeeping/NotificationGenerator.groovy | 71 ++++++++----------- .../housekeeping/NotificationSender.groovy | 8 ++- 2 files changed, 34 insertions(+), 45 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index 0a0afcb86a..ffa6feea4f 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -58,6 +58,8 @@ class NotificationGenerator extends HouseKeeper { PreparedStatement statement ResultSet resultSet + Map> changedInstanceIDsWithComments = [:] + connection = whelk.getStorage().getOuterConnection() connection.setAutoCommit(false) try { @@ -69,9 +71,6 @@ class NotificationGenerator extends HouseKeeper { statement.setTimestamp(2, until) statement.setFetchSize(512) resultSet = statement.executeQuery() - // If both an instance and one of it's dependencies are affected within the same interval, we will - // (without this check) try to generate notifications for said instance twice. - Set affectedInstanceIDs = [] while (resultSet.next()) { String id = resultSet.getString("id") @@ -81,12 +80,35 @@ class NotificationGenerator extends HouseKeeper { // List changeNotes = Arrays.asList( changeNotesArray.getArray() ) won't work. List changeNotes = [] for (Object o : changeNotesArray.getArray()) { - changeNotes.add(o) + if (o != null) + changeNotes.add(o) } - generateObservationsForChangedID(id, changeNotes, from.toInstant(), - until.toInstant(), affectedInstanceIDs) + List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) + dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! + dependers.each { + String dependerID = it[0] + String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) + if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { + if (!changedInstanceIDsWithComments.containsKey(dependerID)) { + changedInstanceIDsWithComments.put(dependerID, []) + } + changedInstanceIDsWithComments[dependerID].addAll(changeNotes) + } + } } + + for (String instanceId : changedInstanceIDsWithComments.keySet()) { + List resultingChangeObservations = generateObservationsForAffectedInstance( + instanceId, changedInstanceIDsWithComments[instanceId], from.toInstant(), until.toInstant()) + + for (Document observation : resultingChangeObservations) { + if (!whelk.createDocument(observation, "NotificationGenerator", "SEK", "none", false)) { + log.error("Failed to create ChangeObservation:\n${observation.getDataAsString()}") + } + } + } + } catch (Throwable e) { status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString() throw e @@ -98,41 +120,6 @@ class NotificationGenerator extends HouseKeeper { } } - /** - * Based on the fact that 'id' has been updated, generate (if the change resulted in a ChangeNotice) - * relevant notification-records - */ - private void generateObservationsForChangedID(String id, List changeNotes, Instant from, Instant until, Set affectedInstanceIDs) { - - List resultingChangeObservations = [] - - List> dependers = whelk.getStorage().followDependers(id, ["itemOf"]) - dependers.add(new Tuple2(id, null)) // This ID too, not _only_ the dependers! - dependers.each { - String dependerID = it[0] - String dependerMainEntityType = whelk.getStorage().getMainEntityTypeBySystemID(dependerID) - if (whelk.getJsonld().isSubClassOf(dependerMainEntityType, "Instance")) { - // If we've not already made an observation for this instance! - if (!affectedInstanceIDs.contains(dependerID)) { - affectedInstanceIDs.add(dependerID) - resultingChangeObservations.addAll( generateObservationsForAffectedInstance(dependerID, changeNotes, from, until) ) - if (resultingChangeObservations.size() > MAX_OBSERVATIONS_PER_CHANGE) { - log.warn("Discarding ChangeObservations for instances related to $id, which was changed. Observations would be too many.") - return - } - } - } - } - - for (Document observation : resultingChangeObservations) { - //System.err.println(" ** Made change observation:\n${observation.getDataAsString()}") - - if (!whelk.createDocument(observation, "NotificationGenerator", "SEK", "none", false)) { - log.error("Failed to create ChangeObservation:\n${observation.getDataAsString()}") - } - } - } - private List generateObservationsForAffectedInstance(String instanceId, List changeNotes, Instant before, Instant after) { List generatedObservations = [] List propertiesToEmbellish = [ @@ -203,7 +190,7 @@ class NotificationGenerator extends HouseKeeper { [ "@id" : mainEntityUri, "@type" : "ChangeObservation", - "about" : ["@id" : Document.BASE_URI.toString() + instanceId], + "concerning" : ["@id" : Document.BASE_URI.toString() + instanceId], "representationBefore" : oldValueEmbedded, "representationAfter" : newValueEmbedded, "category" : ["@id" : categoryUri], diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy index 2f5142a184..52d1838056 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationSender.groovy @@ -104,7 +104,7 @@ class NotificationSender extends HouseKeeper { connection = whelk.getStorage().getOuterConnection() connection.setAutoCommit(false) try { - String sql = "SELECT MAX(created) as lastChange, data#>>'{@graph,1,about,@id}' as instanceUri, ARRAY_AGG(data::text) as data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND created > ? GROUP BY data#>>'{@graph,1,about,@id}';" + String sql = "SELECT MAX(created) as lastChange, data#>>'{@graph,1,concerning,@id}' as instanceUri, ARRAY_AGG(data::text) as data FROM lddb WHERE data#>>'{@graph,1,@type}' = 'ChangeObservation' AND created > ? GROUP BY data#>>'{@graph,1,concerning,@id}';" connection.setAutoCommit(false) statement = connection.prepareStatement(sql) statement.setTimestamp(1, from) @@ -243,9 +243,11 @@ class NotificationSender extends HouseKeeper { sb.append("\n\t\tInnan ändring: " + before["sv"]) sb.append("\n\t\tEfter ändring: " + after["sv"]) - if (observation["comment"]) { + Object comments = Document._get(["@graph", 1, "comment"], observation) + + if (comments instanceof List) { sb.append("\n\t\tTillhörande kommentarer:") - for (String comment : observation["comment"]) + for (String comment : comments) sb.append("\n\t\t\t"+comment) } sb.append("\n\n") From bfb54b9a9dfd7f5d13ce0ffbe689a2e3e54edfe7 Mon Sep 17 00:00:00 2001 From: Jannis Tsiroyannis Date: Thu, 9 Nov 2023 14:38:56 +0100 Subject: [PATCH 58/58] Clean up a bit. --- .../housekeeping/NotificationGenerator.groovy | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy index ffa6feea4f..8e55eb155f 100644 --- a/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy +++ b/housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy @@ -75,9 +75,6 @@ class NotificationGenerator extends HouseKeeper { String id = resultSet.getString("id") Array changeNotesArray = resultSet.getArray("changeNotes") - - // There is some groovy type-nonsense going on with the array types, simply doing - // List changeNotes = Arrays.asList( changeNotesArray.getArray() ) won't work. List changeNotes = [] for (Object o : changeNotesArray.getArray()) { if (o != null) @@ -102,9 +99,11 @@ class NotificationGenerator extends HouseKeeper { List resultingChangeObservations = generateObservationsForAffectedInstance( instanceId, changedInstanceIDsWithComments[instanceId], from.toInstant(), until.toInstant()) - for (Document observation : resultingChangeObservations) { - if (!whelk.createDocument(observation, "NotificationGenerator", "SEK", "none", false)) { - log.error("Failed to create ChangeObservation:\n${observation.getDataAsString()}") + if (resultingChangeObservations.size() <= MAX_OBSERVATIONS_PER_CHANGE) { + for (Document observation : resultingChangeObservations) { + if (!whelk.createDocument(observation, "NotificationGenerator", "SEK", "none", false)) { + log.error("Failed to create ChangeObservation:\n${observation.getDataAsString()}") + } } } } @@ -140,34 +139,40 @@ class NotificationGenerator extends HouseKeeper { if (instanceBeforeChange == null) { // This instance is new, and did not exist at 'before'. return generatedObservations } - historicEmbellish(instanceBeforeChange, propertiesToEmbellish, before); - - // Check for primary contribution changes - { - Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) - Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) - if (contributionsBefore != null && contributionsAfter != null && contributionsBefore instanceof List && contributionsAfter instanceof List) { - for (Object contrBefore : contributionsBefore) { - for (Object contrAfter : contributionsAfter) { - if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution")) { - if (contrBefore["agent"] != null && contrAfter["agent"] != null) { - if ( - contrBefore["agent"]["familyName"] != contrAfter["agent"]["familyName"] || - contrBefore["agent"]["givenName"] != contrAfter["agent"]["givenName"] || - contrBefore["agent"]["lifeSpan"] != contrAfter["agent"]["lifeSpan"] - ) - generatedObservations.add( - makeChangeObservation(instanceId, changeNotes, - "https://id.kb.se/changecategory/primarycontribution", - (Map) contrBefore["agent"], (Map) contrAfter["agent"])) - } + historicEmbellish(instanceBeforeChange, propertiesToEmbellish, before) + + Tuple comparisonResult = primaryContributionChanged(instanceBeforeChange, instanceAfterChange) + if (comparisonResult[0]) { + generatedObservations.add( + makeChangeObservation( + instanceId, changeNotes, "https://id.kb.se/changecategory/primarycontribution", + (Map) comparisonResult[1], (Map) comparisonResult[2]) + ) + } + + return generatedObservations + } + + private static Tuple primaryContributionChanged(Document instanceBeforeChange, Document instanceAfterChange) { + Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data) + Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data) + if (contributionsBefore != null && contributionsAfter != null && contributionsBefore instanceof List && contributionsAfter instanceof List) { + for (Object contrBefore : contributionsBefore) { + for (Object contrAfter : contributionsAfter) { + if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution")) { + if (contrBefore["agent"] != null && contrAfter["agent"] != null) { + if ( + contrBefore["agent"]["familyName"] != contrAfter["agent"]["familyName"] || + contrBefore["agent"]["givenName"] != contrAfter["agent"]["givenName"] || + contrBefore["agent"]["lifeSpan"] != contrAfter["agent"]["lifeSpan"] + ) + return new Tuple(true, contrBefore["agent"], contrAfter["agent"]) } } } } } - - return generatedObservations + return new Tuple(false, null, null) } private Document makeChangeObservation(String instanceId, List changeNotes, String categoryUri, Map oldValue, Map newValue) {