diff --git a/pom.xml b/pom.xml index 6997426..08bae23 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.googlecode.the-fascinator.plugins plugin-harvester-csv - 1.1.3-SNAPSHOT + 1.1.3-UWS-1-SNAPSHOT Fascinator - Plugin - Harvester - CSV organisation @@ -11,18 +11,18 @@ 1.0.3 - 1.1.2 + 1.1.3-UWS-1-SNAPSHOT com.googlecode.the-fascinator fascinator-common - 1.1.2 + 1.1.3-UWS-1-SNAPSHOT com.googlecode.the-fascinator.plugins plugin-storage-ram - 1.1.2 + 1.1.3-UWS-1-SNAPSHOT test diff --git a/src/main/java/com/googlecode/fascinator/harvester/csv/CSVHarvester.java b/src/main/java/com/googlecode/fascinator/harvester/csv/CSVHarvester.java index 63a7b52..a1c9ef3 100644 --- a/src/main/java/com/googlecode/fascinator/harvester/csv/CSVHarvester.java +++ b/src/main/java/com/googlecode/fascinator/harvester/csv/CSVHarvester.java @@ -248,7 +248,7 @@ public String toString() { /** Current row */ private long currentRow; - + /** Whether or not there are more files to harvest */ private boolean hasMore; @@ -388,7 +388,7 @@ public void shutdown() throws HarvesterException { public boolean hasMoreObjects() { return hasMore; } - + /** * Harvest the next batch of rows and return their object IDs. * @@ -523,13 +523,17 @@ private void storeJsonInObject(JsonObject dataJson, JsonObject metaJson, DigitalObject object = null; try { object = getStorage().getObject(oid); + boolean isModified = isModifiedRecord(oid, dataJson); storeJsonInPayload(dataJson, metaJson, object); + object.getMetadata().setProperty("isModified", Boolean.toString(isModified)); } catch (StorageException ex) { // This is going to be brand new try { object = StorageUtils.getDigitalObject(getStorage(), oid); storeJsonInPayload(dataJson, metaJson, object); + // newRecordCount++; + object.getMetadata().setProperty("isNew", "true"); } catch (StorageException ex2) { throw new HarvesterException( "Error creating new digital object: ", ex2); @@ -546,6 +550,33 @@ private void storeJsonInObject(JsonObject dataJson, JsonObject metaJson, } } } + + /** + * Check if this record is going to be modified. + * + * @param oid an object id to check + * @param dataJson an instantiated JSON object containing data to store + * @returns true if modifies, false otherwise + */ + private boolean isModifiedRecord(String oid, JsonObject dataJson) + { + DigitalObject object = null; + JsonObject jo = null; + try { + object = getStorage().getObject(oid); + Payload p = object.getPayload(payloadId); + jo = new JsonSimple(p.open()).getObject("data"); + return !jo.equals(dataJson); + + } catch (StorageException e) { + log.error("Error retrieving object from storage: ", e); + } catch (HarvesterException e) { + log.error("Error getting harvested object' flag: ", e); + } catch (IOException e) { + log.error("Error accessing the storage' flag: ", e); + } + return false; + } /** * Store the processed data and metadata in a payload