Skip to content

Commit

Permalink
Add the NotificationCleaner and do a little refactoring.
Browse files Browse the repository at this point in the history
  • Loading branch information
jannistsiroyannis committed Nov 16, 2023
1 parent 10247be commit c7a1dd6
Show file tree
Hide file tree
Showing 4 changed files with 295 additions and 217 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package whelk.housekeeping

import groovy.transform.CompileStatic
import groovy.util.logging.Log4j2
import whelk.Whelk

import java.sql.Connection
import java.sql.PreparedStatement
import java.sql.ResultSet
import java.sql.SQLType
import java.sql.Timestamp
import java.sql.Types
import java.time.Instant
import java.time.ZoneOffset
import java.time.temporal.ChronoUnit

@CompileStatic
@Log4j2
class NotificationCleaner extends HouseKeeper {
private static final int DAYS_TO_KEEP_INACTIVE = 14
private String status = "OK"
private final Whelk whelk

public NotificationCleaner(Whelk whelk) {
this.whelk = whelk
}

public String getName() {
return "Notifications cleaner"
}

public String getStatusDescription() {
return status
}

public String getCronSchedule() {
return "* * * * *"
}

public void trigger() {
Connection connection
PreparedStatement statement
ResultSet resultSet

connection = whelk.getStorage().getOuterConnection()
connection.setAutoCommit(false)
try {
Timestamp from = Timestamp.from(Instant.now().minus(DAYS_TO_KEEP_INACTIVE, ChronoUnit.DAYS))

Set<String> typesToClean = whelk.getJsonld().getSubClasses("AdministrativeNotice")
typesToClean.add("AdministrativeNotice")

String sql = "SELECT id FROM lddb WHERE collection = 'none' AND data#>>'{@graph,1,@type}' = ANY (?) AND modified < ? AND created < ?;"
connection.setAutoCommit(false)
statement = connection.prepareStatement(sql)
statement.setArray(1, connection.createArrayOf("TEXT", typesToClean as String[]))
statement.setTimestamp(2, from)
statement.setTimestamp(3, from)
statement.setFetchSize(512)
resultSet = statement.executeQuery()
while (resultSet.next()) {
String id = resultSet.getString("id")
whelk.remove(id, "NotificationGenerator", "SEK")
}
} catch (Throwable e) {
status = "Failed with:\n" + e + "\nat:\n" + e.getStackTrace().toString()
throw e
} finally {
connection.close()
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ class NotificationGenerator extends HouseKeeper {
Tuple comparisonResult

// Primary Contribution
comparisonResult = primaryContributionChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.primaryContributionChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -153,7 +153,7 @@ class NotificationGenerator extends HouseKeeper {
}

// Intended Audience
comparisonResult = intendedAudienceChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.intendedAudienceChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -163,7 +163,7 @@ class NotificationGenerator extends HouseKeeper {
}

// Main Title
comparisonResult = mainTitleChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.mainTitleChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -173,7 +173,7 @@ class NotificationGenerator extends HouseKeeper {
}

// Primary Title
comparisonResult = primaryTitleChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.primaryTitleChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -183,7 +183,7 @@ class NotificationGenerator extends HouseKeeper {
}

// Primary Publication
comparisonResult = primaryPublicationChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.primaryPublicationChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -193,7 +193,7 @@ class NotificationGenerator extends HouseKeeper {
}

// DDC classification
comparisonResult = DDCChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.DDCChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -203,7 +203,7 @@ class NotificationGenerator extends HouseKeeper {
}

// SAB classification
comparisonResult = SABChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.SABChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -213,7 +213,7 @@ class NotificationGenerator extends HouseKeeper {
}

// Serial relation
comparisonResult = SerialRelationChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.SerialRelationChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -223,7 +223,7 @@ class NotificationGenerator extends HouseKeeper {
}

// Serial termination
comparisonResult = SerialTerminationChanged(instanceBeforeChange, instanceAfterChange)
comparisonResult = NotificationRules.SerialTerminationChanged(instanceBeforeChange, instanceAfterChange)
if (comparisonResult[0]) {
generatedObservations.add(
makeChangeObservation(
Expand All @@ -235,213 +235,6 @@ class NotificationGenerator extends HouseKeeper {
return generatedObservations
}

private static Tuple primaryContributionChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object contributionsAfter = Document._get(["mainEntity", "instanceOf", "contribution"], instanceAfterChange.data)
Object contributionsBefore = Document._get(["mainEntity", "instanceOf", "contribution"], instanceBeforeChange.data)
if (contributionsBefore != null && contributionsAfter != null && contributionsBefore instanceof List && contributionsAfter instanceof List) {
for (Object contrBefore : contributionsBefore) {
for (Object contrAfter : contributionsAfter) {
if (contrBefore["@type"].equals("PrimaryContribution") && contrAfter["@type"].equals("PrimaryContribution")) {
if (contrBefore["agent"] != null && contrAfter["agent"] != null) {
if (
contrBefore["agent"]["familyName"] != contrAfter["agent"]["familyName"] ||
contrBefore["agent"]["givenName"] != contrAfter["agent"]["givenName"] ||
contrBefore["agent"]["lifeSpan"] != contrAfter["agent"]["lifeSpan"]
)
return new Tuple(true, contrBefore["agent"], contrAfter["agent"])
}
}
}
}
}
return new Tuple(false, null, null)
}

private static Tuple primaryPublicationChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object publicationsAfter = Document._get(["mainEntity", "publication"], instanceAfterChange.data)
Object publicationsBefore = Document._get(["mainEntity", "publication"], instanceBeforeChange.data)
if (publicationsBefore != null && publicationsAfter != null && publicationsBefore instanceof List && publicationsAfter instanceof List) {
for (Object pubBefore : publicationsBefore) {
for (Object pubAfter : publicationsAfter) {
if (pubBefore["@type"].equals("PrimaryPublication") && pubAfter["@type"].equals("PrimaryPublication")) {
if (!pubBefore["year"].equals(pubAfter["year"])) {
return new Tuple(true, pubBefore["year"], pubAfter["year"])
}
}
}
}
}
return new Tuple(false, null, null)
}

private static Tuple SerialRelationChanged(Document instanceBeforeChange, Document instanceAfterChange) {

if (!Document._get(["issuanceType"], instanceBeforeChange.data).equals("Serial"))
return new Tuple(false, null, null)
if (!Document._get(["issuanceType"], instanceAfterChange.data).equals("Serial"))
return new Tuple(false, null, null)

Object precededBefore = Document._get(["precededBy"], instanceBeforeChange.data)
Object precededAfter = Document._get(["precededBy"], instanceAfterChange.data)
if (precededBefore != null && precededAfter != null && precededBefore instanceof List && precededAfter instanceof List) {
if (precededAfter as Set != precededBefore as Set)
return new Tuple(true, precededBefore, precededAfter)
}

Object succeededBefore = Document._get(["succeededBy"], instanceBeforeChange.data)
Object succeededAfter = Document._get(["succeededBy"], instanceAfterChange.data)
if (succeededBefore != null && succeededAfter != null && succeededBefore instanceof List && succeededAfter instanceof List) {
if (succeededAfter as Set != succeededBefore as Set)
return new Tuple(true, succeededBefore, succeededAfter)
}

return new Tuple(false, null, null)
}

private static Tuple SerialTerminationChanged(Document instanceBeforeChange, Document instanceAfterChange) {

if (!Document._get(["issuanceType"], instanceBeforeChange.data).equals("Serial"))
return new Tuple(false, null, null)
if (!Document._get(["issuanceType"], instanceAfterChange.data).equals("Serial"))
return new Tuple(false, null, null)

Object publicationsBefore = Document._get(["publication"], instanceBeforeChange.data)
Object publicationsAfter = Document._get(["publication"], instanceAfterChange.data)

if (publicationsBefore instanceof List && publicationsAfter instanceof List &&
publicationsBefore.size() == publicationsAfter.size()) {
List beforeList = (List) publicationsBefore
List afterList = (List) publicationsAfter
for (int i = 0; i < beforeList.size(); ++i)
if (!beforeList[i]["endYear"].equals(afterList[i]["endYear"])) {
return new Tuple(true, beforeList[i]["endYear"], afterList[i]["endYear"])
}
}

return new Tuple(false, null, null)
}

private static Tuple intendedAudienceChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object valueBefore = Document._get(["mainEntity", "instanceOf", "intendedAudience"], instanceBeforeChange.data)
Object valueAfter = Document._get(["mainEntity", "instanceOf", "intendedAudience"], instanceAfterChange.data)

if (valueBefore != null && valueAfter != null && valueBefore instanceof List && valueAfter instanceof List) {
if (valueAfter as Set != valueBefore as Set)
return new Tuple(true, valueBefore, valueAfter)
}
return new Tuple(false, null, null)
}

private static Tuple mainTitleChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object titlesBefore = Document._get(["mainEntity", "hasTitle"], instanceBeforeChange.data)
Object titlesAfter = Document._get(["mainEntity", "hasTitle"], instanceAfterChange.data)

Map oldMainTitle = null
Map newMainTitle = null

if (titlesBefore != null && titlesAfter != null && titlesBefore instanceof List && titlesAfter instanceof List) {

for (Object oBefore : titlesBefore) {
Map titleBefore = (Map) oBefore
if (titleBefore["mainTitle"])
oldMainTitle = titleBefore
}

for (Object oAfter : titlesAfter) {
Map titleAfter = (Map) oAfter
if (titleAfter["mainTitle"])
newMainTitle = titleAfter
}

if (newMainTitle != null && oldMainTitle != null && !newMainTitle.equals(oldMainTitle))
return new Tuple(true, oldMainTitle, newMainTitle)
}
return new Tuple(false, null, null)
}

private static Tuple primaryTitleChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object titlesBefore = Document._get(["mainEntity", "hasTitle"], instanceBeforeChange.data)
Object titlesAfter = Document._get(["mainEntity", "hasTitle"], instanceAfterChange.data)

Map oldPrimaryTitle = null
Map newPrimaryTitle = null

if (titlesBefore != null && titlesAfter != null && titlesBefore instanceof List && titlesAfter instanceof List) {

for (Object oBefore : titlesBefore) {
Map titleBefore = (Map) oBefore
if (titleBefore["@type"] && titleBefore["@type"] == "Title")
oldPrimaryTitle = titleBefore
}

for (Object oAfter : titlesAfter) {
Map titleAfter = (Map) oAfter
if (titleAfter["@type"] && titleAfter["@type"] == "Title")
newPrimaryTitle = titleAfter
}

if (newPrimaryTitle != null && oldPrimaryTitle != null && !newPrimaryTitle.equals(oldPrimaryTitle))
return new Tuple(true, oldPrimaryTitle, newPrimaryTitle)
}
return new Tuple(false, null, null)
}

private static Tuple DDCChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object classificationsBefore = Document._get(["mainEntity", "instanceOf", "classification"], instanceBeforeChange.data)
Object classificationsAfter = Document._get(["mainEntity", "instanceOf", "classification"], instanceAfterChange.data)

Map oldDDC = null
Map newDDC = null

if (classificationsBefore != null && classificationsAfter != null && classificationsBefore instanceof List && classificationsAfter instanceof List) {

for (Object oBefore : classificationsBefore) {
Map classificationBefore = (Map) oBefore
if (classificationBefore["@type"] && classificationBefore["@type"] == "ClassificationDdc")
oldDDC = classificationBefore
}

for (Object oAfter : classificationsAfter) {
Map classificationAfter = (Map) oAfter
if (classificationAfter["@type"] && classificationAfter["@type"] == "ClassificationDdc")
newDDC = classificationAfter
}

if (newDDC != null && oldDDC != null && !newDDC.equals(oldDDC))
return new Tuple(true, oldDDC, newDDC)
}
return new Tuple(false, null, null)
}

private static Tuple SABChanged(Document instanceBeforeChange, Document instanceAfterChange) {
Object classificationsBefore = Document._get(["mainEntity", "instanceOf", "classification"], instanceBeforeChange.data)
Object classificationsAfter = Document._get(["mainEntity", "instanceOf", "classification"], instanceAfterChange.data)

Map oldSAB = null
Map newSAB = null

if (classificationsBefore != null && classificationsAfter != null && classificationsBefore instanceof List && classificationsAfter instanceof List) {

for (Object oBefore : classificationsBefore) {
Map classificationBefore = (Map) oBefore
if (classificationBefore["inScheme"] && classificationBefore["inScheme"]["code"] &&
classificationBefore["inScheme"]["code"] == "kssb")
oldSAB = classificationBefore
}

for (Object oAfter : classificationsAfter) {
Map classificationAfter = (Map) oAfter
if (classificationAfter["inScheme"] && classificationAfter["inScheme"]["code"] &&
classificationAfter["inScheme"]["code"] == "kssb")
newSAB = classificationAfter
}

if (newSAB != null && oldSAB != null && !newSAB.equals(oldSAB))
return new Tuple(true, oldSAB, newSAB)
}
return new Tuple(false, null, null)
}

private Document makeChangeObservation(String instanceId, List changeNotes, String categoryUri, Object oldValue, Object newValue, String agentId) {
String newId = IdGenerator.generate()
String metadataUri = Document.BASE_URI.toString() + newId
Expand Down
Loading

0 comments on commit c7a1dd6

Please sign in to comment.