Merge pull request #12 from GeoCat/cswharvester-extractidentifierfiel…

…dname

CSW Harvester - extract identifier field name

services/csw-harvester/src/main/java/geocat/csw/CSWMetadata.java

@@ -12,20 +12,24 @@ public class CSWMetadata {
     private boolean lookForNestedDiscoveryService;
     private String filter;
 
+    private String identifierFieldName;
+
     public CSWMetadata(String harvesterId,
                        long endpointId,
                        int numberOfExpectedRecords,
                        String getRecordsUrl,
                        List<List<String>> nestedGetCapUrls,
                        String filter,
-                       boolean lookForNestedDiscoveryService) {
+                       boolean lookForNestedDiscoveryService,
+                       String identifierFieldName) {
         this.harvesterId = harvesterId;
         this.endpointId = endpointId;
         this.numberOfExpectedRecords = numberOfExpectedRecords;
         this.getRecordsUrl = getRecordsUrl;
         this.nestedGetCapUrls = nestedGetCapUrls;
         this.filter = filter;
         this.lookForNestedDiscoveryService = lookForNestedDiscoveryService;
+        this.identifierFieldName = identifierFieldName;
     }
 
     public boolean isLookForNestedDiscoveryService() {
@@ -83,4 +87,12 @@ public long getEndpointId() {
     public void setEndpointId(long endpointId) {
         this.endpointId = endpointId;
     }
+
+    public String getIdentifierFieldName() {
+        return identifierFieldName;
+    }
+
+    public void setIdentifierFieldName(String identifierFieldName) {
+        this.identifierFieldName = identifierFieldName;
+    }
 }

services/csw-harvester/src/main/java/geocat/csw/CSWService.java

@@ -56,6 +56,8 @@ public CSWMetadata getMetadata(CSWEndPointDetectedEvent endPointDetectedEvent) t
         String getRecordsUrl = endpoint_extractGetRecordsURL(endPointDetectedEvent);
         int expectedNumberOfRecords = endpoint_numberOfRecordsMatchingFilter(endPointDetectedEvent, getRecordsUrl);
 
+        String identifierFieldName= endpoint_identifierFieldName(endPointDetectedEvent);
+
         List<List<String>> nestedGetCaps = new ArrayList<>();
         if (endPointDetectedEvent.isLookForNestedDiscoveryService())
             nestedGetCaps = endpoint_nestedDiscovery(endPointDetectedEvent, getRecordsUrl);
@@ -66,7 +68,8 @@ public CSWMetadata getMetadata(CSWEndPointDetectedEvent endPointDetectedEvent) t
                 getRecordsUrl,
                 nestedGetCaps,
                 filter,
-                endPointDetectedEvent.isLookForNestedDiscoveryService());
+                endPointDetectedEvent.isLookForNestedDiscoveryService(),
+                identifierFieldName);
         return result;
     }
 
@@ -93,8 +96,15 @@ private String endpoint_extractGetRecordsURL(CSWEndPointDetectedEvent endPointDe
         return cswGetCapHandler.extractGetRecordsURL(getcapXmlString);
     }
 
-    public String GetRecords(String url, String filter, int startRecord, int endRecord, Boolean doNotSort) throws Exception {
+    private String endpoint_identifierFieldName(CSWEndPointDetectedEvent endPointDetectedEvent) throws Exception {
+        logger.debug("requesting GetCapabilities from URL=" + endPointDetectedEvent.getUrl());
+
+        String getcapXmlString = cswEngine.GetCapabilities(endPointDetectedEvent.getUrl());
+        return cswGetCapHandler.extractIdentifierFieldName(getcapXmlString);
+    }
+
+    public String GetRecords(String url, String filter, int startRecord, int endRecord, Boolean doNotSort, String identifierFieldName) throws Exception {
         boolean _doNotSort = doNotSort != null && doNotSort;
-        return cswEngine.GetRecords(url, ogcFilterService.getRecordsXML(filter, startRecord, endRecord, _doNotSort));
+        return cswEngine.GetRecords(url, ogcFilterService.getRecordsXML(filter, startRecord, endRecord, _doNotSort, identifierFieldName));
     }
 }

services/csw-harvester/src/main/java/geocat/csw/csw/CSWGetCapHandler.java

@@ -58,4 +58,20 @@ public String extractGetRecordsURL(String getCapResponseXML) throws Exception {
 
         return u.toString();
     }
+
+    public String extractIdentifierFieldName(String getCapResponseXML) throws Exception {
+        Document getCapDoc = XMLTools.parseXML(getCapResponseXML);
+
+        String identifierFieldName = "";
+
+        NodeList nodes = XMLTools.xpath_nodeset(getCapDoc, "/Capabilities/OperationsMetadata/Operation[@name='GetRecords']/Constraint[@name='SupportedISOQueryables']/Value");
+        for (int i = 0; i < nodes.getLength(); i++) {
+            if (nodes.item(i).getTextContent().toLowerCase().equals("identifier") ||
+                    nodes.item(i).getTextContent().toLowerCase().contains(":identifier")) {
+                identifierFieldName = nodes.item(i).getNodeValue();
+                break;
+            }
+        }
+        return identifierFieldName;
+    }
 }

services/csw-harvester/src/main/java/geocat/csw/csw/OGCFilterService.java

@@ -2,6 +2,7 @@
 
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
 
@@ -43,7 +44,7 @@ public class OGCFilterService {
 
     public static String ORDERBY = " <ogc:SortBy>\n" +
             "            <ogc:SortProperty>\n" +
-            "                <ogc:PropertyName>Identifier</ogc:PropertyName>\n" +
+            "                <ogc:PropertyName>PUT_ORDERBYFIELDNAME_HERE</ogc:PropertyName>\n" +
             "                <ogc:SortOrder>DESC</ogc:SortOrder>\n" +
             "            </ogc:SortProperty>\n" +
             "        </ogc:SortBy>\n";
@@ -124,7 +125,7 @@ public String getGetdiscoveryXml(String filter) throws Exception {
         return GETDISCOVERY_XML.replace("PUT_FILTER_HERE", fullFilter);
     }
 
-    public String getRecordsXML(String filter, int startRecord, int endRecord, boolean doNotSort) {
+    public String getRecordsXML(String filter, int startRecord, int endRecord, boolean doNotSort, String identifierFieldName) {
         String xml;
         int nrecords = endRecord - startRecord + 1;
         if ((filter != null) && (!filter.isEmpty())) {
@@ -136,10 +137,11 @@ public String getRecordsXML(String filter, int startRecord, int endRecord, boole
             xml = GETRECORDS_NO_FILTER_XML.replace("PUT_START_POSITION_HERE", Integer.toString(startRecord))
                     .replace("PUT_MAX_RECORDS_HERE", Integer.toString(nrecords));
         }
-        if (doNotSort)
+        if (doNotSort || !StringUtils.hasLength(identifierFieldName)) {
             xml = xml.replace("PUT_ORDERBY_HERE", "");
-        else
-            xml = xml.replace("PUT_ORDERBY_HERE", ORDERBY);
+        } else {
+            xml = xml.replace("PUT_ORDERBY_HERE", ORDERBY.replace("PUT_ORDERBYFIELDNAME_HERE", identifierFieldName));
+        }
         return xml;
     }
 

services/csw-harvester/src/main/java/geocat/database/entities/EndpointJob.java

@@ -39,6 +39,9 @@ public class EndpointJob {
     @Enumerated(EnumType.STRING)
     private EndpointJobState state;
 
+    @Column(name = "identifier_field_name")
+    private String identifierFieldName;
+
     @PrePersist
     private void onInsert() {
         this.createTimeUTC = ZonedDateTime.now(ZoneId.of("UTC"));
@@ -122,4 +125,12 @@ public String getUrl() {
     public void setUrl(String url) {
         this.url = url;
     }
+
+    public String getIdentifierFieldName() {
+        return identifierFieldName;
+    }
+
+    public void setIdentifierFieldName(String identifierFieldName) {
+        this.identifierFieldName = identifierFieldName;
+    }
 }

services/csw-harvester/src/main/java/geocat/database/service/DatabaseUpdateService.java

@@ -55,6 +55,7 @@ public List<CSWEndPointDetectedEvent> updateDatabase(CSWMetadata cswMetadata) {
         EndpointJob endpointJob = endpointJobRepo.findById(cswMetadata.getEndpointId()).get();
         endpointJob.setExpectedNumberOfRecords(cswMetadata.getNumberOfExpectedRecords());
         endpointJob.setUrlGetRecords(cswMetadata.getGetRecordsUrl());
+        endpointJob.setIdentifierFieldName(cswMetadata.getIdentifierFieldName());
 
         List<CSWEndPointDetectedEvent> result = createCSWEndPointDetectedEvents(cswMetadata);
         // endpointJob.setState(EndpointJobState.WORK_DETERMINED);

services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_GetRecordsCommand.java

@@ -64,14 +64,16 @@ public EventProcessor_GetRecordsCommand() {
     public EventProcessor_GetRecordsCommand externalProcessing() throws Exception {
 
         GetRecordsCommand e = getInitiatingEvent();
-        String xml = cswService.GetRecords(e.getGetRecordsURL(), e.getFilter(), e.getStartRecordNumber(), e.getEndRecordNumber(), e.getDoNotSort());
+
+        EndpointJob endpointJob = endpointJobService.getById(e.getEndPointId());
+
+        String xml = cswService.GetRecords(e.getGetRecordsURL(), e.getFilter(), e.getStartRecordNumber(), e.getEndRecordNumber(), e.getDoNotSort(), endpointJob.getIdentifierFieldName());
         //logger.debug("starting to parse xml");
         Document xmlParsed = MetadataExploderService.parseXML(xml);
         //logger.debug("finish parse xml");
 
         GetRecordsResponseInfo info = new GetRecordsResponseInfo(xmlParsed);
         HarvestJob harvestJob = harvestJobService.getById(e.getHarvesterId()).get();
-        EndpointJob endpointJob = endpointJobService.getById(e.getEndPointId());
         RecordSet recordSet = recordSetService.getById(e.getRecordSetId());
 
         //logger.debug("eval...");