Skip and log invalid docs

whelktool/README.md

@@ -22,20 +22,30 @@ Options
 ```
 $ java -jar build/libs/whelktool.jar --help
 usage: whelktool [options] <SCRIPT>
- -a,--allow-loud            Allow scripts to do loud modifications.
- -d,--dry-run               Do not save any modifications.
- -h,--help                  Print this help message and exit.
- -I,--skip-index            Do not index any changes, only write to
-                            storage.
- -l,--limit <LIMIT>         Amount of documents to process.
- -n,--stats-num-ids <arg>   Number of ids to print per entry in
-                            STATISTICS.txt.
- -r,--report <REPORT-DIR>   Directory where reports are written (defaults
-                            to "reports").
- -s,--step                  Change one document at a time, prompting to
-                            continue.
- -T,--no-threads            Do not use threads to parallellize batch
-                            processing.
+ -a,--allow-loud                    Allow scripts to do loud
+                                    modifications.
+ -d,--dry-run                       Do not save any modifications.
+ -h,--help                          Print this help message and exit.
+ -I,--skip-index                    Do not index any changes, only write
+                                    to storage.
+ -idchg,--allow-id-removal          [UNSAFE] Allow script to remove
+                                    document ids, e.g. sameAs.
+ -l,--limit <LIMIT>                 Amount of documents to process.
+ -n,--stats-num-ids <arg>           Number of ids to print per entry in
+                                    STATISTICS.txt.
+ -p,--parameters <PARAMETER-FILE>   Path to JSON file with parameters to
+                                    script
+ -r,--report <REPORT-DIR>           Directory where reports are written
+                                    (defaults to "reports").
+ -s,--step                          Change one document at a time,
+                                    prompting to continue.
+ -T,--no-threads                    Do not use threads to parallellize
+                                    batch processing.
+ -t,--num-threads <N>               Override default number of threads
+                                    (32).
+ -v,--validation <MODE>             [UNSAFE] Set JSON-LD validation mode.
+                                    Defaults to ON. Possible values:
+                                    ON/OFF/LOG-ONLY
 ```
 
 * Use `--dry-run` to test your script without writing any changes.

whelktool/src/main/groovy/whelk/datatool/WhelkTool.groovy

@@ -9,7 +9,6 @@ import whelk.IdGenerator
 import whelk.JsonLd
 import whelk.JsonLdValidator
 import whelk.Whelk
-import whelk.datatool.form.ModifiedThing
 import whelk.datatool.form.Transform
 import whelk.datatool.util.IdLoader
 import whelk.exception.StaleUpdateException
@@ -21,12 +20,10 @@ import whelk.util.DocumentUtil
 import whelk.util.LegacyIntegrationTools
 import whelk.util.Statistics
 
-import javax.print.Doc
 import javax.script.Bindings
 import javax.script.CompiledScript
 import javax.script.ScriptEngineManager
 import javax.script.SimpleBindings
-import java.nio.charset.StandardCharsets
 import java.time.ZonedDateTime
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ConcurrentLinkedQueue
@@ -41,7 +38,6 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicInteger
 
 import static java.util.concurrent.TimeUnit.SECONDS
-import static whelk.JsonLd.RECORD_KEY
 import static whelk.util.Jackson.mapper
 
 class WhelkTool {
@@ -50,6 +46,7 @@ class WhelkTool {
     static final int DEFAULT_STATS_NUM_IDS = 3
     public static final String MAIN_LOG_NAME = "MAIN.txt"
     public static final String ERROR_LOG_NAME = "ERRORS.txt"
+    public static final String INVALID_LOG_NAME = "INVALID.txt"
     public static final String MODIFIED_LOG_NAME = "MODIFIED.txt"
     public static final String CREATED_LOG_NAME = "CREATED.txt"
     public static final String DELETED_LOG_NAME = "DELETED.txt"
@@ -68,6 +65,8 @@ class WhelkTool {
     File reportsDir
     PrintWriter mainLog
     PrintWriter errorLog
+
+    PrintWriter invalidLog
     PrintWriter modifiedLog
     PrintWriter createdLog
     PrintWriter deletedLog
@@ -90,7 +89,14 @@ class WhelkTool {
 
     boolean allowLoud
     boolean allowIdRemoval
-    boolean skipValidation = false
+
+    enum ValidationMode {
+        ON,
+        OFF,
+        LOG_ONLY
+    }
+
+    ValidationMode validationMode = ValidationMode.ON
 
     Throwable errorDetected
 
@@ -123,7 +129,7 @@ class WhelkTool {
         reportsDir.mkdirs()
         mainLog = new PrintWriter(new File(reportsDir, MAIN_LOG_NAME))
         errorLog = new PrintWriter(new File(reportsDir, ERROR_LOG_NAME))
-
+        invalidLog = new PrintWriter(new File(reportsDir, INVALID_LOG_NAME))
         def modifiedLogFile = new File(reportsDir, MODIFIED_LOG_NAME)
         modifiedLog = new PrintWriter(modifiedLogFile)
         def createdLogFile = new File(reportsDir, CREATED_LOG_NAME)
@@ -562,7 +568,7 @@ class WhelkTool {
         doc.setGenerationDate(new Date())
         doc.setGenerationProcess(item.generationProcess ?: script.scriptJobUri)
 
-        if (!skipValidation) {
+        if (validationMode in [ValidationMode.ON, ValidationMode.LOG_ONLY]) {
             validateJsonLd(doc)
         }
 
@@ -582,7 +588,7 @@ class WhelkTool {
         doc.setGenerationDate(new Date())
         doc.setGenerationProcess(item.generationProcess ?: script.scriptJobUri)
 
-        if (!skipValidation) {
+        if (validationMode in [ValidationMode.ON, ValidationMode.LOG_ONLY]) {
             validateJsonLd(doc)
         }
 
@@ -600,7 +606,13 @@ class WhelkTool {
     private void validateJsonLd(Document doc) {
         List<JsonLdValidator.Error> errors = validator.validate(doc.data, doc.getLegacyCollection(whelk.jsonld))
         if (errors) {
-            throw new Exception("Invalid JSON-LD. Errors: ${errors.collect{ it.toMap() }}")
+            String msg = "Invalid JSON-LD in document ${doc.completeId}. Errors: ${errors.collect { it.toMap() }}"
+            if (validationMode == ValidationMode.ON) {
+                throw new Exception(msg)
+            } else if (validationMode == ValidationMode.LOG_ONLY) {
+                invalidLog.println(doc.shortId)
+                errorLog.println(msg)
+            }
         }
     }
 
@@ -707,7 +719,7 @@ class WhelkTool {
         if (limit > -1) log "  limit: $limit"
         if (allowLoud) log "  allowLoud"
         if (allowIdRemoval) log "  allowIdRemoval"
-        if (skipValidation) log " skipValidation"
+        log "  validation: ${validationMode.name()}"
         log()
 
         bindings = createMainBindings()
@@ -773,7 +785,8 @@ class WhelkTool {
         cli.l(longOpt: 'limit', args: 1, argName: 'LIMIT', 'Amount of documents to process.')
         cli.a(longOpt: 'allow-loud', 'Allow scripts to do loud modifications.')
         cli.idchg(longOpt: 'allow-id-removal', '[UNSAFE] Allow script to remove document ids, e.g. sameAs.')
-        cli.sv(longOpt: 'skip-validation', '[UNSAFE] Skip JSON-LD validation before saving to database.')
+        cli.v(longOpt: 'validation', args: 1, argName: 'MODE', '[UNSAFE] Set JSON-LD validation mode. Defaults to ON.' +
+                ' Possible values: ON/OFF/LOG-ONLY')
         cli.n(longOpt: 'stats-num-ids', args: 1, 'Number of ids to print per entry in STATISTICS.txt.')
         cli.p(longOpt: 'parameters', args: 1, argName: 'PARAMETER-FILE', 'Path to JSON file with parameters to script')
 
@@ -810,7 +823,7 @@ class WhelkTool {
         tool.limit = options.l ? Integer.parseInt(options.l) : -1
         tool.allowLoud = options.a
         tool.allowIdRemoval = options.idchg
-        tool.skipValidation = options.sv
+        tool.validationMode = parseValidationMode(options.v) ?: tool.validationMode
         try {
             tool.run()
         } catch (Exception e) {
@@ -819,6 +832,15 @@ class WhelkTool {
         }
     }
 
+    private static ValidationMode parseValidationMode(String arg) {
+        for (vm in ValidationMode.getEnumConstants()) {
+            if (arg.toUpperCase() == vm.name()) {
+                return vm
+            }
+        }
+        return null
+    }
+
     void recordChange(Document before, Document after, int number) {
         //if (recordingLimit >= 0 && number > recordingLimit) {
         if (recordingLimit >= 0 && recordedChanges.size() > recordingLimit) {

whelktool/src/main/java/whelk/datatool/bulkchange/BulkJob.java

@@ -32,6 +32,7 @@
 import static whelk.datatool.WhelkTool.DELETED_LOG_NAME;
 import static whelk.datatool.WhelkTool.MAIN_LOG_NAME;
 import static whelk.datatool.WhelkTool.MODIFIED_LOG_NAME;
+import static whelk.datatool.WhelkTool.ValidationMode.LOG_ONLY;
 import static whelk.datatool.bulkchange.BulkJobDocument.JOB_TYPE;
 import static whelk.datatool.bulkchange.BulkJobDocument.Status.Completed;
 import static whelk.datatool.bulkchange.BulkJobDocument.Status.Failed;
@@ -163,6 +164,7 @@ protected WhelkTool buildWhelkTool(BulkJobDocument jobDoc) throws IOException {
         tool.setDefaultChangedBy(jobDoc.getChangeAgentId());
         tool.setAllowLoud(jobDoc.shouldUpdateModifiedTimestamp());
         tool.setNoThreads(false);
+        tool.setValidationMode(LOG_ONLY);
 
         return tool;
     }

whelktool/src/main/java/whelk/datatool/bulkchange/BulkPreviewJob.java

@@ -11,6 +11,7 @@
 import java.nio.file.Files;
 import java.util.List;
 
+import static whelk.datatool.WhelkTool.ValidationMode.LOG_ONLY;
 import static whelk.datatool.bulkchange.BulkJobDocument.JOB_TYPE;
 
 public class BulkPreviewJob extends BulkJob {
@@ -30,6 +31,7 @@ public BulkPreviewJob(Whelk whelk, String id) throws IOException {
         tool.setDryRun(true);
         tool.setRecordChanges(true);
         tool.setRecordingLimit(RECORD_MAX_ITEMS);
+        tool.setValidationMode(LOG_ONLY);
         tool.setLogger(log);
     }