Merge remote-tracking branch 'origin/dev' into feature/issue1058

grails-app/conf/application.groovy

@@ -1633,9 +1633,9 @@ if (!darwinCore.termsGroupedByClass) {
             ],
             "MeasurementOrFact"   : [
                     [
-                            "name"      : "measurementsorfacts",
+                            "name"      : "measurementsOrFacts",
                             "substitute": { record, params ->
-                                record.measurementsorfacts ?: []
+                                record.measurementsOrFacts ?: []
                             },
                             order: [
                                     "eventID",
@@ -1654,7 +1654,7 @@ if (!darwinCore.termsGroupedByClass) {
                     [
                             "name"      : "multimedia",
                             "substitute": { record, params ->
-                                record?.multimedia?.collect { mediaRecord ->
+                                record?.multimedia?.findResults { mediaRecord ->
                                     if (mediaRecord.documentId) {
                                         Document document = Document.findByDocumentIdAndStatusNotEqual(mediaRecord.documentId, DELETED)
                                         if (document) {

grails-app/controllers/au/org/ala/ecodata/AdminController.groovy

@@ -740,7 +740,8 @@ class AdminController {
     def regenerateRecordsForALAHarvestableProjects() {
         def result = [:]
         try {
-            recordService.regenerateRecordsForBioCollectProjects()
+            List projects = params.projectId?.split(',')?.toList()
+            recordService.regenerateRecordsForBioCollectProjectsOrProjectList(projects)
             result.message = "Submitted regeneration of records"
         }
         catch (Exception e) {

grails-app/domain/au/org/ala/ecodata/Record.groovy

@@ -45,6 +45,7 @@ class Record {
     String outputId
     String json
     Integer outputItemId
+    List measurementsOrFacts = []
     String status = ACTIVE
 
     static transients = ['recordNumber']
@@ -78,6 +79,7 @@ class Record {
         name nullable: true
         vernacularName nullable: true
         scientificName nullable: true
+        measurementsOrFacts nullable: true
     }
 
     String getRecordNumber(sightingsUrl){

grails-app/services/au/org/ala/ecodata/ActivityService.groovy

@@ -165,14 +165,20 @@ class ActivityService {
         }
     }
 
-    List findAllForProjectId(id, levelOfDetail = [], includeDeleted = false) {
+    List findAllForProjectId(id, levelOfDetail = [], includeDeleted = false, includeExternalActivities = false) {
         List activities
         if (includeDeleted) {
             activities = Activity.findAllByProjectId(id).collect {toMap(it, levelOfDetail)}
         }
-        else {
+        else if (includeExternalActivities) {
             activities = Activity.findAllByProjectIdAndStatus(id, ACTIVE).collect { toMap(it, levelOfDetail) }
         }
+        else {
+            // By specifying externalIds:null we are excluding Monitor activities from the search as they can't
+            // contribute to the output scores and for some projects there are a lot of them and they can have
+            // a very large amount of data associated with them.
+            activities = Activity.findAllByProjectIdAndStatusAndExternalIds(id, ACTIVE, null).collect { toMap(it, levelOfDetail) }
+        }
         activities
     }
 

grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy

@@ -1146,9 +1146,9 @@ class ElasticSearchService {
                     site.remove('geoIndex')
                     site
                 }
-                projectMap.activities = activityService.findAllForProjectId(project.projectId, LevelOfDetail.NO_OUTPUTS.name())
+                projectMap.activities = activityService.findAllForProjectId(project.projectId, LevelOfDetail.NO_OUTPUTS.name(), false, true)
             } else {
-                projectMap.activities = activityService.findAllForProjectId(project.projectId, LevelOfDetail.NO_OUTPUTS.name()).collect{[type:it.type]}
+                projectMap.activities = activityService.findAllForProjectId(project.projectId, LevelOfDetail.NO_OUTPUTS.name(), false, true).collect{[type:it.type]}
             }
 
             // If we don't flatten these values into the root of the project, they are not currently usable by

src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy

@@ -236,8 +236,8 @@ class HarvestControllerSpec extends Specification {
                     // check MeasurementOrFact.csv
                     CSVReader readerCSV = new CSVReader(new StringReader(content))
                     List<String[]> lines = readerCSV.readAll()
-                    assert lines[0] == ["eventID","occurrenceID","measurementValue","measurementAccuracy","measurementUnit","measurementUnitID","measurementType","measurementTypeID"]
-                    assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","distance from source","http://qudt.org/vocab/quantitykind/Number"]
+                    assert lines[0] == ["eventID","occurrenceID","measurementValue","measurementAccuracy","measurementUnit","measurementUnitID","measurementType","measurementTypeID", "measurementID"]
+                    assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","distance from source","http://qudt.org/vocab/quantitykind/Number",""]
                     assert lines.size() ==2
                     break
             }

src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy

@@ -12,7 +12,7 @@ class ListConverter implements RecordFieldConverter {
         int index = 0
 
         // delegate the conversion of each column in each row to a specific converter for the column type
-        data[outputMetadata.name].each { row ->
+        data?.get(outputMetadata.name)?.each { row ->
             if (row == null) {
                 return
             }

src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy

@@ -173,18 +173,21 @@ class RecordConverter {
             records << baseRecord
         }
 
-        // Add measurements or facts only when event core archive is being created.
-        if (recordGeneration) {
-            records.each {
-                it.remove(PROP_MEASUREMENTS_OR_FACTS)
-            }
-        }
+        convertMeasurementsOrFactsToList(records)
         end = System.currentTimeMillis()
         log.debug("Time in milliseconds to convert nested data model - ${end - start}")
         // We are now left with a list of one or more Maps, where each Map contains all the fields for an individual Record.
         records
     }
 
+    static List convertMeasurementsOrFactsToList (List records) {
+        records?.each { record ->
+            record[PROP_MEASUREMENTS_OR_FACTS] = record[PROP_MEASUREMENTS_OR_FACTS]?.collect { it.value } ?: []
+        }
+
+        records
+    }
+
     /**
      * Return a new Map with the union of source and additional giving precedence to values from additional
      * If the same key already exists in target it will be overridden
@@ -289,17 +292,21 @@ class RecordConverter {
         dwcFields
     }
 
-    static void updateSpeciesIdToMeasurements(List measurements, String id) {
-        measurements?.each {
-            if(!it.occurrenceID)
-                it.occurrenceID = id
+    static void updateSpeciesIdToMeasurements(Map measurements, String id) {
+        measurements?.each { key, value ->
+            if (!value.occurrenceID) {
+                value.occurrenceID = id
+                key.occurrenceID = id
+            }
         }
     }
 
-    static void updateEventIdToMeasurements(List measurements, String id) {
-        measurements?.each {
-            if(!it.eventID)
-                it.eventID = id
+    static void updateEventIdToMeasurements(Map measurements, String id) {
+        measurements?.each { key, value ->
+            if (!value.eventID) {
+                value.eventID = id
+                key.eventID = id
+            }
         }
     }
 
@@ -332,6 +339,10 @@ class RecordConverter {
 
                 result[entry.key].addAll(entry.value)
             }
+            if (entry.value instanceof Map) {
+                result[entry.key] = result[entry.key] ?: [:]
+                result[entry.key] << entry.value
+            }
             else {
                 result[entry.key] = entry.value
             }
@@ -340,17 +351,4 @@ class RecordConverter {
         result
     }
 
-    /**
-     * Removes for duplicate entries. A duplicate entry is when all values in a map are the same.
-     * [a: 'b', c: 'd'] & [a: 'b', c: 'd'] are duplicates.
-     * [a: 'b', c: 'd'] & [a: 'b', c: 'e'] are not duplicates.
-     * @param measurements
-     * @return
-     */
-    static List removeDuplicates (List measurements) {
-        measurements?.groupBy {
-            it.values().toString() }.collect { key, values ->
-            values?.get(0)
-        }
-    }
 }

src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy

@@ -25,7 +25,7 @@ trait RecordFieldConverter {
     static String DWC_MEASUREMENT_ACCURACY = "measurementAccuracy"
     static String DWC_MEASUREMENT_UNIT = "measurementUnit"
     static String DWC_MEASUREMENT_UNIT_ID = "measurementUnitID"
-    static String PROP_MEASUREMENTS_OR_FACTS = "measurementsorfacts"
+    static String PROP_MEASUREMENTS_OR_FACTS = "measurementsOrFacts"
 
     abstract List<Map> convert(Map data)
 
@@ -73,11 +73,14 @@ trait RecordFieldConverter {
         // It dwcAttribute measurementValue is taken from output data.
         // If measurementType is provided in metadata, its event core value is created by binding metadata value and
         // output data. Rest of the attributes are provided by metadata.
+        if(!dwcMappings.containsKey(DWC_MEASUREMENT_VALUE))
+            return fields
+
         String fieldName = dwcMappings[DWC_MEASUREMENT_VALUE]
         def value = data[fieldName]
-        if (dwcMappings.containsKey(DWC_MEASUREMENT_VALUE) && ![null, ""].contains(value)) {
+        if (value !in [null, ""]) {
             if (!fields[PROP_MEASUREMENTS_OR_FACTS])
-                fields[PROP_MEASUREMENTS_OR_FACTS] = []
+                fields[PROP_MEASUREMENTS_OR_FACTS] = [:]
 
             Map measurement = [:]
             measurement[DWC_MEASUREMENT_VALUE] = value
@@ -99,7 +102,8 @@ trait RecordFieldConverter {
                 measurement[DWC_MEASUREMENT_UNIT_ID] = metadata[DWC_MEASUREMENT_UNIT_ID]
             }
 
-            fields[PROP_MEASUREMENTS_OR_FACTS].add(measurement)
+            // to remove/overwrite duplicate measurements
+            fields[PROP_MEASUREMENTS_OR_FACTS][measurement] = measurement
         }
 
         fields
@@ -135,7 +139,6 @@ trait RecordFieldConverter {
             return value
         }
         catch (Exception ex) {
-            log.error(ex.message)
             data.remove('context')
             return defaultValue == null ? expression : defaultValue
         }

src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy

@@ -413,6 +413,7 @@ class RecordConverterSpec extends Specification {
 
     def "converter should add measurements or facts to the record field set"() {
         setup:
+        List measurementsOrFacts
         Project project = new Project(projectId: "project1")
         Organisation organisation = new Organisation(orgId: "org1")
         Site site = new Site(siteId: "site1")
@@ -447,17 +448,19 @@ class RecordConverterSpec extends Specification {
 
         when:
         List<Map> fieldsets = RecordConverter.convertRecords(project, organisation, site, projectActivity, activity, output, submittedData, outputMetadata, false)
+        measurementsOrFacts = fieldsets[0].measurementsOrFacts
 
         then:
         fieldsets.size() == 1
         fieldsets[0].attribute1 == "fieldValue1"
-        fieldsets[0].measurementsorfacts.size() == 1
-        fieldsets[0].measurementsorfacts[0].measurementValue == 10.056
-        fieldsets[0].measurementsorfacts[0].measurementType == "distance from source"
-        fieldsets[0].measurementsorfacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
-        fieldsets[0].measurementsorfacts[0].measurementAccuracy == "0.001"
-        fieldsets[0].measurementsorfacts[0].measurementUnit == "m"
-        fieldsets[0].measurementsorfacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M"
+        fieldsets[0].measurementsOrFacts.size() == 1
+
+        measurementsOrFacts[0].measurementValue == 10.056
+        measurementsOrFacts[0].measurementType == "distance from source"
+        measurementsOrFacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
+        measurementsOrFacts[0].measurementAccuracy == "0.001"
+        measurementsOrFacts[0].measurementUnit == "m"
+        fieldsets[0].measurementsOrFacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M"
 
         when: "the measurement is associated with a species and non-species"
         outputMetadata = [
@@ -541,26 +544,26 @@ class RecordConverterSpec extends Specification {
         fieldsets[0].name == "scientificName1"
         fieldsets[0].outputSpeciesId == "speciesFieldId1"
         fieldsets[0].occurrenceID == "speciesFieldId1"
-        fieldsets[0].measurementsorfacts.size() == 3
-        fieldsets[0].measurementsorfacts[0].eventID == "act1"
-        fieldsets[0].measurementsorfacts[0].measurementValue == 10.056
-        fieldsets[0].measurementsorfacts[0].measurementType == "distance from source"
-        fieldsets[0].measurementsorfacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
-        fieldsets[0].measurementsorfacts[0].measurementAccuracy == "0.001"
-        fieldsets[0].measurementsorfacts[0].measurementUnit == "m"
-        fieldsets[0].measurementsorfacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M"
-        fieldsets[0].measurementsorfacts[1].measurementValue == 21.056
-        fieldsets[0].measurementsorfacts[1].measurementType == "distance from source"
-        fieldsets[0].measurementsorfacts[1].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
-        fieldsets[0].measurementsorfacts[1].measurementAccuracy == "0.001"
-        fieldsets[0].measurementsorfacts[1].measurementUnit == "m"
-        fieldsets[0].measurementsorfacts[1].measurementUnitID == "http://qudt.org/vocab/unit/M"
-        fieldsets[0].measurementsorfacts[2].measurementValue == 23.056
-        fieldsets[0].measurementsorfacts[2].measurementType == "distance from source"
-        fieldsets[0].measurementsorfacts[2].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
-        fieldsets[0].measurementsorfacts[2].measurementAccuracy == "0.001"
-        fieldsets[0].measurementsorfacts[2].measurementUnit == "m"
-        fieldsets[0].measurementsorfacts[2].measurementUnitID == "http://qudt.org/vocab/unit/M"
+        fieldsets[0].measurementsOrFacts.size() == 2
+        fieldsets[0].measurementsOrFacts[0].eventID == "act1"
+        fieldsets[0].measurementsOrFacts[0].measurementValue == 10.056
+        fieldsets[0].measurementsOrFacts[0].measurementType == "distance from source"
+        fieldsets[0].measurementsOrFacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
+        fieldsets[0].measurementsOrFacts[0].measurementAccuracy == "0.001"
+        fieldsets[0].measurementsOrFacts[0].measurementUnit == "m"
+        fieldsets[0].measurementsOrFacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M"
+        fieldsets[0].measurementsOrFacts[1].measurementValue == 21.056
+        fieldsets[0].measurementsOrFacts[1].measurementType == "distance from source"
+        fieldsets[0].measurementsOrFacts[1].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
+        fieldsets[0].measurementsOrFacts[1].measurementAccuracy == "0.001"
+        fieldsets[0].measurementsOrFacts[1].measurementUnit == "m"
+        fieldsets[0].measurementsOrFacts[1].measurementUnitID == "http://qudt.org/vocab/unit/M"
+        fieldsets[1].measurementsOrFacts[1].measurementValue == 23.056
+        fieldsets[1].measurementsOrFacts[1].measurementType == "distance from source"
+        fieldsets[1].measurementsOrFacts[1].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number"
+        fieldsets[1].measurementsOrFacts[1].measurementAccuracy == "0.001"
+        fieldsets[1].measurementsOrFacts[1].measurementUnit == "m"
+        fieldsets[1].measurementsOrFacts[1].measurementUnitID == "http://qudt.org/vocab/unit/M"
         fieldsets[1].attribute1 == 1
         fieldsets[1].attribute2 == "present"
         fieldsets[1].scientificName == "scientificName2"
@@ -569,6 +572,6 @@ class RecordConverterSpec extends Specification {
         fieldsets[1].name == "scientificName2"
         fieldsets[1].outputSpeciesId == "speciesFieldId2"
         fieldsets[1].occurrenceID == "speciesFieldId2"
-        fieldsets[1].measurementsorfacts.size() == 3
+        fieldsets[1].measurementsOrFacts.size() == 2
     }
 }