diff --git a/README.md b/README.md index b1ddf7b00..db010c4af 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ To report an issue, please go to [issues](https://github.com/Esri/geoportal-serv The nature of the Harvester application is, as the name suggests, to harvest metadata from whatever web endpoints it is provided. The list(s) of endpoints to download metadata from can also be provided by external entities over the internet. Neither the metadata being harvested nor the list(s) of endpoints provided by external entities are vetted or checked by the Harvester. **Users who wish to limit the scope of the Harvester's reach should configure the network or machine where the Harvester is located with allow lists or deny lists of web endpoints to prevent the Harvester from reaching undesirable locations.** ## Releases and Downloads -- 2.7.0 - June 13, 2021, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.7.0) for release notes and downloads. +- 2.7.1 - June 13, 2021, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.7.1) for release notes and downloads. - 2.6.5 - July 13, 2021, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.6.5) for release notes and downloads. - 2.6.4 - July 8, 2020, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.6.4) for release notes and downloads. diff --git a/geoportal-SDK/geoportal-harvester-api-base/pom.xml b/geoportal-SDK/geoportal-harvester-api-base/pom.xml index c966bd0c9..af19d12e3 100644 --- a/geoportal-SDK/geoportal-harvester-api-base/pom.xml +++ b/geoportal-SDK/geoportal-harvester-api-base/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-SDK - 2.7.0 + 2.7.1 harvester-api-base Esri :: Geoportal Server :: Harvester :: Api Base diff --git a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java index e5e0cd874..6a2af73b4 100644 --- a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java +++ b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java @@ -75,6 +75,10 @@ public Set getContentType() { public String getId() { return baseRef.getId(); } + @Override + public String getTitle() { + return baseRef.getTitle(); + } @Override public String getFetchableId() { diff --git a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java index bde0a2ccf..f1efa0b7c 100644 --- a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java +++ b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java @@ -34,6 +34,7 @@ public class SimpleDataReference implements DataReference { private final URI brokerUri; private final String brokerName; private final String id; + private String title = ""; private final Date lastModifiedDate; private final URI sourceUri; private final String inputBrokerRef; @@ -54,6 +55,16 @@ public class SimpleDataReference implements DataReference { * @param taskRef task reference of null if ad-hoc */ public SimpleDataReference(URI brokerUri, String brokerName, String id, Date lastModifiedDate, URI sourceUri, String inputBrokerRef, String taskRef) { + this.brokerUri = brokerUri; + this.brokerName = brokerName; + this.id = id; + this.lastModifiedDate = lastModifiedDate; + this.sourceUri = sourceUri; + this.inputBrokerRef = inputBrokerRef; + this.taskRef = taskRef; + } + + public SimpleDataReference(URI brokerUri, String brokerName, String id, Date lastModifiedDate, URI sourceUri, String inputBrokerRef, String taskRef,String title) { this.brokerUri = brokerUri; this.brokerName = brokerName; this.id = id; @@ -61,6 +72,7 @@ public SimpleDataReference(URI brokerUri, String brokerName, String id, Date las this.sourceUri = sourceUri; this.inputBrokerRef = inputBrokerRef; this.taskRef = taskRef; + this.title = title; } /** @@ -101,6 +113,11 @@ public Date getLastModifiedDate() { public URI getSourceUri() { return sourceUri; } + + @Override + public String getTitle() { + return title; + } @Override public byte[] getContent(MimeType...mimeType) throws IOException { diff --git a/geoportal-SDK/geoportal-harvester-api/pom.xml b/geoportal-SDK/geoportal-harvester-api/pom.xml index 3ee20895f..8ca7330a3 100644 --- a/geoportal-SDK/geoportal-harvester-api/pom.xml +++ b/geoportal-SDK/geoportal-harvester-api/pom.xml @@ -4,12 +4,12 @@ geoportal-SDK com.esri.geoportal - 2.7.0 + 2.7.1 harvester-api Esri :: Geoportal Server :: Harvester :: Api Definitions of all basic elements of the Harvester (interfaces, final classes, etc.). - 2.7.0 + 2.7.1 jar diff --git a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java index b6cb24743..5839f0216 100644 --- a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java +++ b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java @@ -86,4 +86,6 @@ public interface DataReference extends Serializable, DataContent { * @return task reference of null if ad-hoc */ String getTaskRef(); + + String getTitle(); } diff --git a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java index c8343bdfd..7ac4d169e 100644 --- a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java +++ b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java @@ -81,7 +81,10 @@ public static enum ArgumentType { /** temporal type */ temporal, /** periodical */ - periodical + periodical, + button, + hidden + } /** @@ -224,7 +227,7 @@ public String toString() { } } - /** + /** * String argument. */ public static class StringArgument extends ArgumentBase { @@ -272,6 +275,44 @@ public ArgumentType getType() { } } + /** + * Button argument. + */ + public static class ButtonArgument extends ArgumentBase { + + /** + * Creates instance of the argument. + * @param name type + * @param label label + * @param required true if argument is required + */ + public ButtonArgument(String name, String label, boolean required) { + super(name, label, required); + } + @Override + public ArgumentType getType() { + return ArgumentType.button; + } + } + + /** + * Button argument. + */ + public static class HiddenArgument extends ArgumentBase { + /** + * Creates instance of the argument. + * @param name type + * @param label label + */ + public HiddenArgument(String name, String label) { + super(name, label); + } + @Override + public ArgumentType getType() { + return ArgumentType.hidden; + } + } + /** * String argument. */ diff --git a/geoportal-SDK/pom.xml b/geoportal-SDK/pom.xml index 3652bc1e3..2088d0bb2 100644 --- a/geoportal-SDK/pom.xml +++ b/geoportal-SDK/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-SDK pom diff --git a/geoportal-application/geoportal-harvester-cli/pom.xml b/geoportal-application/geoportal-harvester-cli/pom.xml index 6fc3d6921..9d78426e8 100644 --- a/geoportal-application/geoportal-harvester-cli/pom.xml +++ b/geoportal-application/geoportal-harvester-cli/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-application - 2.7.0 + 2.7.1 geoportal-harvester-cli jar diff --git a/geoportal-application/geoportal-harvester-engine/pom.xml b/geoportal-application/geoportal-harvester-engine/pom.xml index b92d56e90..efd007cf9 100644 --- a/geoportal-application/geoportal-harvester-engine/pom.xml +++ b/geoportal-application/geoportal-harvester-engine/pom.xml @@ -4,7 +4,7 @@ geoportal-application com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-harvester-engine Esri :: Geoportal Server :: Harvester :: Application :: Engine diff --git a/geoportal-application/geoportal-harvester-war/pom.xml b/geoportal-application/geoportal-harvester-war/pom.xml index 6aa1c7726..00aa94749 100644 --- a/geoportal-application/geoportal-harvester-war/pom.xml +++ b/geoportal-application/geoportal-harvester-war/pom.xml @@ -4,7 +4,7 @@ geoportal-application com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-harvester-war war @@ -13,7 +13,7 @@ ${project.build.directory}/endorsed - 5.3.24 + 5.3.27 5.3.10.RELEASE @@ -67,7 +67,7 @@ com.h2database h2 - 2.1.210 + 2.2.220 diff --git a/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java b/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java index 4dd5e4424..32b82f0c7 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java +++ b/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java @@ -18,6 +18,9 @@ import static com.esri.geoportal.commons.utils.CrlfUtils.formatForLog; import com.esri.geoportal.harvester.api.defs.UITemplate; import com.esri.geoportal.harvester.engine.services.Engine; +import com.esri.geoportal.harvester.engine.services.TemplatesService; +import java.util.List; +import java.util.Locale; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -181,6 +184,8 @@ public UITemplate[] listInboundConnectors() { @RequestMapping(value = "/rest/harvester/connectors/outbound", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public UITemplate[] listOutboundConnectors() { LOG.debug(String.format("GET /rest/harvester/connectors/outbound")); + TemplatesService srv1 = engine.getTemplatesService(); + List temp2= srv1.getOutboundConnectorTemplates(Locale.US); return engine.getTemplatesService().getOutboundConnectorTemplates(LocaleContextHolder.getLocale()).toArray(new UITemplate[0]); } diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js index 6fe09ebe1..29f3bf535 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js @@ -28,14 +28,18 @@ define(["dojo/_base/declare", "dijit/Dialog", "dijit/ConfirmDialog", "hrv/rest/Brokers", - "hrv/ui/brokers/BrokerEditorPane" + "hrv/ui/brokers/BrokerEditorPane", + "esri/IdentityManager", + "esri/arcgis/Portal", + "esri/config", ], function(declare, _WidgetBase,_TemplatedMixin,_WidgetsInTemplateMixin, i18n,template, lang,string,topic,on,json, Dialog,ConfirmDialog, - BrokersREST,BrokerEditorPane + BrokersREST,BrokerEditorPane, + esriId, arcgisPortal,esriConfig ){ return declare([_WidgetBase, _TemplatedMixin, _WidgetsInTemplateMixin],{ @@ -71,23 +75,50 @@ define(["dojo/_base/declare", // listen to "submit" button click this.own(on(brokerEditorPane,"submit",lang.hitch(this, function(evt){ - var brokerDefinition = evt.brokerDefinition; - - // use API to update broker - BrokersREST.update(brokerDefinition.uuid,json.stringify(brokerDefinition)).then( - lang.hitch({brokerEditorPane: brokerEditorPane, brokerEditorDialog: brokerEditorDialog, self: this},function(){ - topic.publish("msg"); // clear any former errors - this.brokerEditorDialog.destroy(); - this.brokerEditorPane.destroy(); - this.self.load(); - }), - lang.hitch(this,function(error){ - console.debug(error); - topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); - }) - ); - }))); - + var brokerDefinition = evt.brokerDefinition; + var brokerDefinitionProp = evt.brokerDefinition.properties; + var portalUrl = brokerDefinitionProp["agp-host-url"] ; + + if(brokerDefinitionProp["agp-oauth"]=== "true"){ + esriId.getCredential(portalUrl,{oAuthPopupConfirmation:false}).then( + lang.hitch(this,function(credential){ + var token = credential.token; + brokerDefinitionProp["agp-token"]= token; + + // use API to update broker + BrokersREST.update(brokerDefinition.uuid,json.stringify(brokerDefinition)).then( + lang.hitch({brokerEditorPane: brokerEditorPane, brokerEditorDialog: brokerEditorDialog, self: this},function(){ + topic.publish("msg"); // clear any former errors + this.brokerEditorDialog.destroy(); + this.brokerEditorPane.destroy(); + //TODO reload Broker + // this.self.load(); + }), + lang.hitch(this,function(error){ + console.debug(error); + topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); + }) + ); + })); + } + else + { + brokerDefinitionProp["agp-token"]= ""; + // use API to update broker + BrokersREST.update(brokerDefinition.uuid,json.stringify(brokerDefinition)).then( + lang.hitch({brokerEditorPane: brokerEditorPane, brokerEditorDialog: brokerEditorDialog, self: this},function(){ + topic.publish("msg"); // clear any former errors + this.brokerEditorDialog.destroy(); + this.brokerEditorPane.destroy(); + this.self.load(); + }), + lang.hitch(this,function(error){ + console.debug(error); + topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); + })); + } + }))); + brokerEditorDialog.show(); }, diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js index 888d804e0..b6bd25697 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js @@ -108,6 +108,10 @@ define(["dojo/_base/declare", this.updateArgumentsForm(this.connectorTemplates[type].arguments); }, + onOAuth:function() + { + alert("hello"); + }, _onSubmit: function() { if (this.formWidget.validate()) { var values = this.formWidget.getValues(); diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js index a60e3ea21..fed1444ff 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js @@ -26,17 +26,19 @@ define(["dojo/_base/declare", "dijit/form/Select", "dijit/form/ValidationTextBox", "dijit/form/CheckBox", + "dijit/form/TextBox", "dijit/form/TimeTextBox", "dijit/form/RadioButton", "dijit/form/NumberTextBox", "dijit/form/Textarea", "dijit/form/Form", + "dijit/form/Button", "dojox/html/entities", "hrv/utils/TextScrambler" ], function(declare,i18n, lang,array,domConstruct,domAttr,html,number, - Select,ValidationTextBox,CheckBox,TimeTextBox,RadioButton,NumberTextBox,Textarea,Form, + Select,ValidationTextBox,CheckBox,TextBox,TimeTextBox,RadioButton,NumberTextBox,Textarea,Form,Button, entities,TextScrambler ){ @@ -61,7 +63,8 @@ define(["dojo/_base/declare", renderArgument: function(rootNode,arg) { var argNode = domConstruct.create("div",{class: "h-editor-line"},rootNode); - var titleNode = domConstruct.create("span",{innerHTML: arg.label+":", class: "h-editor-argname"},argNode); + if(arg.label!=="hidden") + var titleNode = domConstruct.create("span",{innerHTML: arg.label+":", class: "h-editor-argname"},argNode); var placeholderWrapper = domConstruct.create("span",{class: "h-editor-argctrl"},argNode); var placeholderNode = domConstruct.create("span",null,placeholderWrapper); @@ -79,6 +82,8 @@ define(["dojo/_base/declare", case "temporal": return this.renderTime(placeholderNode,arg); case "periodical": return this.renderPeriod(placeholderNode,arg); case "integer": return this.renderInteger(placeholderNode, arg); + case "button": return this.renderButton(placeholderNode, arg); + case "hidden": return this.renderHidden(placeholderNode, arg); default: console.error("Unsupported argument type:", arg.type); return { @@ -126,16 +131,16 @@ define(["dojo/_base/declare", renderText: function(placeholderNode,arg) { var input = new Textarea({ name: arg.name, - required: arg.required + required: arg.required }).placeAt(placeholderNode); input.name = arg.name; - if (arg.defaultValue!=null) { + if (arg.defaultValue!==null) { input.set("value", arg.defaultValue); } input.startup(); return { init: function(values) { - input.set("value", values[arg.name]!=null? values[arg.name]: arg.defaultValue); + input.set("value", values[arg.name]!==null? values[arg.name]: arg.defaultValue); }, read: function(values) { values[arg.name] = input.get("value"); @@ -145,6 +150,47 @@ define(["dojo/_base/declare", } }; }, + + renderButton: function (placeholderNode, arg) { + var input = new Button({ + name: arg.name, + required: arg.required + }).placeAt(placeholderNode); + + input.set("label", arg.label); + input.startup(); + + return { + init: function (values) { + }, + read: function (values) { + }, + destroy: function () { + input.destroy(); + } + } + }, + renderHidden: function (placeholderNode, arg) { + var input = new TextBox({ + name: arg.name, + type: "hidden" + }); + + input.set("value", ""); + input.startup(); + + return { + init: function (values) { + input.set("value", Number(values[arg.name])); + }, + read: function (values) { + values[arg.name] = input.get("value"); + }, + destroy: function () { + input.destroy(); + } + }; + }, renderInteger: function (placeholderNode, arg) { var input = new NumberTextBox({ diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html index e6b539f6a..119d17e13 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html @@ -4,7 +4,7 @@
-
Ver. 2.7.0
+
Ver. 2.7.1
diff --git a/geoportal-application/pom.xml b/geoportal-application/pom.xml index 2d7641be6..eb338b8e9 100644 --- a/geoportal-application/pom.xml +++ b/geoportal-application/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-application pom diff --git a/geoportal-commons/geoportal-commons-agp-client/pom.xml b/geoportal-commons/geoportal-commons-agp-client/pom.xml index a51da88bd..64348fb61 100644 --- a/geoportal-commons/geoportal-commons-agp-client/pom.xml +++ b/geoportal-commons/geoportal-commons-agp-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-agp-client Esri :: Geoportal Server :: Commons :: ArcGIS Portal Client diff --git a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java index 62de43bdc..95b1afb68 100644 --- a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java +++ b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java @@ -29,6 +29,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -306,6 +308,49 @@ public String writeItemMetadata(String itemId, String metadataXML, String token) throw ex; } } + + /** + * Writes layer metadata. + * @param resourceURL + * @param fileToUpload metadata + * @param token token + * @return true metadata if and only if update successful + * @throws URISyntaxException if invalid URL + * @throws IOException if operation fails + */ + public boolean writeSubLayerMetadata(String resourceURL, String fileToUpload, String token) throws IOException, URISyntaxException { + URIBuilder builder = new URIBuilder(resourceURL); + HttpPost req = new HttpPost(builder.build()); + + Map params = new HashMap<>(); + params.put("f", "json"); + params.put("token", token); + params.put("metadataUploadFormat", "xml"); + params.put("overwrite", "true"); + params.put("metadata",fileToUpload); + + try { + HttpEntity entity = createEntity(params); + req.setEntity(entity); + + try (CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { + if (httpResponse.getStatusLine().getStatusCode()>=400) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + String responseContent = IOUtils.toString(contentStream, "UTF-8"); + LOG.debug(" writeSubLayerMetadata "+responseContent); + if (responseContent.contains("error")) { + return false; + } + } + + } catch (IOException ex) { + LOG.error("Error writeSubLayerMetadata " +ex.getMessage(),ex); + return false; + } + + return true; + } /** * Sharing item. @@ -518,6 +563,25 @@ public TokenResponse generateToken(int minutes) throws URISyntaxException, IOExc return execute(req,TokenResponse.class); } + public TokenResponse generateToken(int minutes, String serverUrl,String token) throws URISyntaxException, IOException { + HttpPost req = new HttpPost(generateTokenUri()); + + HashMap params = new HashMap<>(); + params.put("f", "json"); + if (credentials != null) { + params.put("username", StringUtils.trimToEmpty(credentials.getUserName())); + params.put("password", StringUtils.trimToEmpty(credentials.getPassword())); + } + params.put("client", "requestip"); + params.put("expiration", Integer.toString(minutes)); + params.put("serverUrl", serverUrl); + params.put("token",token ); + + req.setEntity(createEntity(params)); + + return execute(req,TokenResponse.class); + } + private Map makeStdParams(String title, String description, ItemType itemType, URL thumbnailUrl, Double [] extent, String [] typeKeywords, String [] tags, String token) { HashMap params = new HashMap<>(); params.put("f", "json"); @@ -706,7 +770,7 @@ private String execute(HttpUriRequest req, Integer redirectDepth) throws IOExcep return execute(newReq, ++redirectDepth); } catch (IOException | URISyntaxException e) { - LOG.debug("Error executing request", e); + LOG.error("Error executing request", e); throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); } } diff --git a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java index 1e966e2b1..100f736e6 100644 --- a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java +++ b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java @@ -19,7 +19,7 @@ * Item entry. */ public final class ItemEntry { - public String id; + public String id; public String owner; public long created; public long modified; diff --git a/geoportal-commons/geoportal-commons-ags-client/pom.xml b/geoportal-commons/geoportal-commons-ags-client/pom.xml index 914f57949..a46acd570 100644 --- a/geoportal-commons/geoportal-commons-ags-client/pom.xml +++ b/geoportal-commons/geoportal-commons-ags-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-ags-client Esri :: Geoportal Server :: Commons :: ArcGIS Server Client diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java index 2c4d0a94a..db59418a5 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java @@ -177,6 +177,19 @@ public ServerResponse readServiceInformation(URL url) throws IOException { response.url = url.toExternalForm(); response.json = responseContent; response.itemInfo = readItemInfo(new URL(url + "/info/itemInfo")); + + response.hasMetadata = false; + response.metadataXML = ""; + String metadataURL = url + "/info/metadata"; + HttpGet getXML = new HttpGet(metadataURL); + try (CloseableHttpResponse httpResponseXML = httpClient.execute(getXML); InputStream contentStreamXML = httpResponseXML.getEntity().getContent();) { + if (httpResponseXML.getStatusLine().getStatusCode()<400) { + String responseContentXML = IOUtils.toString(contentStreamXML, "UTF-8"); + response.metadataXML = responseContentXML; + response.hasMetadata = true; + } + } + return response; } } @@ -201,6 +214,7 @@ public ItemInfo readItemInfo(URL url) throws IOException { mapper.configure(Feature.ALLOW_NON_NUMERIC_NUMBERS, true); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); ItemInfo response = mapper.readValue(responseContent, ItemInfo.class); + return response; } } @@ -228,6 +242,17 @@ public LayerInfo readLayerInformation(String folder, ServiceInfo si, LayerRef lR mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); LayerInfo response = mapper.readValue(responseContent, LayerInfo.class); + if (response.hasMetadata) { + HttpGet getXML = new HttpGet(url + String.format("/metadata", "text/xml")); + try (CloseableHttpResponse httpResponseXML = httpClient.execute(getXML); InputStream contentStreamXML = httpResponseXML.getEntity().getContent();) { + if (httpResponseXML.getStatusLine().getStatusCode()>=400) { + throw new HttpResponseException(httpResponseXML.getStatusLine().getStatusCode(), httpResponseXML.getStatusLine().getReasonPhrase()); + } + String responseContentXML = IOUtils.toString(contentStreamXML, "UTF-8"); + response.metadataXML = responseContentXML; + } + + } response.url = url; response.json = responseContent; return response; diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java index c24006f45..56d1a3fad 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java @@ -36,4 +36,6 @@ public final class ItemInfo { public String spatialReference; public String accessInformation; public String licenseInfo; + public boolean hasMetadata; + public String metadataXML; } diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java index e92ba1f27..a77eece4e 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java @@ -26,4 +26,6 @@ public class LayerInfo { public String description; public ExtentInfo extent; public String json; + public boolean hasMetadata; + public String metadataXML; } diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java index ecbac3721..4158abbd6 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java @@ -23,6 +23,8 @@ public final class ServerResponse { public String url; public String json; + public boolean hasMetadata; + public String metadataXML; public String mapName; public String serviceDescription; @@ -36,6 +38,6 @@ public final class ServerResponse { @Override public String toString() { - return String.format("{ \"mapName\": \"%s\", \"serviceDescription\": \"%s\", \"spatialReference\": %s, \"initialExtent\": %s, \"fullExtent\": %s}", mapName, serviceDescription, spatialReference, initialExtent, fullExtent); + return String.format("{ \"mapName\": \"%s\", \"serviceDescription\": \"%s\", \"spatialReference\": %s, \"initialExtent\": %s, \"fullExtent\": %s, \"metadata\": %s}", mapName, serviceDescription, spatialReference, initialExtent, fullExtent, metadataXML); } } diff --git a/geoportal-commons/geoportal-commons-ckan-client/pom.xml b/geoportal-commons/geoportal-commons-ckan-client/pom.xml index f58dd4d37..3bc44bb0f 100644 --- a/geoportal-commons/geoportal-commons-ckan-client/pom.xml +++ b/geoportal-commons/geoportal-commons-ckan-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-ckan-client Esri :: Geoportal Server :: Commons :: CKAN Lightweight Client diff --git a/geoportal-commons/geoportal-commons-constants/pom.xml b/geoportal-commons/geoportal-commons-constants/pom.xml index 768800826..49631bfb7 100644 --- a/geoportal-commons/geoportal-commons-constants/pom.xml +++ b/geoportal-commons/geoportal-commons-constants/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-constants jar diff --git a/geoportal-commons/geoportal-commons-csw-client/pom.xml b/geoportal-commons/geoportal-commons-csw-client/pom.xml index fb4521c0f..4fafbf194 100644 --- a/geoportal-commons/geoportal-commons-csw-client/pom.xml +++ b/geoportal-commons/geoportal-commons-csw-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-csw-client Esri :: Geoportal Server :: Commons :: Csw Client diff --git a/geoportal-commons/geoportal-commons-dcat-client/pom.xml b/geoportal-commons/geoportal-commons-dcat-client/pom.xml index a184aa498..c771281c1 100644 --- a/geoportal-commons/geoportal-commons-dcat-client/pom.xml +++ b/geoportal-commons/geoportal-commons-dcat-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-dcat-client jar diff --git a/geoportal-commons/geoportal-commons-doc/pom.xml b/geoportal-commons/geoportal-commons-doc/pom.xml index bce527e5d..c823f28ee 100644 --- a/geoportal-commons/geoportal-commons-doc/pom.xml +++ b/geoportal-commons/geoportal-commons-doc/pom.xml @@ -6,7 +6,7 @@ geoportal-commons com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-commons-doc jar diff --git a/geoportal-commons/geoportal-commons-geometry/pom.xml b/geoportal-commons/geoportal-commons-geometry/pom.xml index b15705407..cca39d022 100644 --- a/geoportal-commons/geoportal-commons-geometry/pom.xml +++ b/geoportal-commons/geoportal-commons-geometry/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-geometry Esri :: Geoportal Server :: Commons :: Geometry Utils diff --git a/geoportal-commons/geoportal-commons-gpt-client/pom.xml b/geoportal-commons/geoportal-commons-gpt-client/pom.xml index 930df7352..9e9a1a515 100644 --- a/geoportal-commons/geoportal-commons-gpt-client/pom.xml +++ b/geoportal-commons/geoportal-commons-gpt-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-gpt-client Esri :: Geoportal Server :: Commons :: Geoportal Rest Client diff --git a/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java b/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java index d7dbe5820..f0aca05df 100644 --- a/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java +++ b/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java @@ -76,768 +76,773 @@ */ public class Client implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(Client.class); - private static final int BATCH_SIZE = 500; - - private static final String DEFAULT_INDEX = "metadata"; - private static final String REST_ITEM_URL = "rest/metadata/item"; - private static final String ELASTIC_SEARCH_URL = "elastic/{metadata}/item/_search"; - private static final String ELASTIC_SCROLL_URL = "elastic/_search/scroll"; - private static final String TOKEN_URL = "oauth/token"; - - private final CloseableHttpClient httpClient; - private final URL url; - private final SimpleCredentials cred; - private final String index; - private final String collectionsFieldName; - - private TokenInfo tokenInfo; - - private final ObjectMapper mapper = new ObjectMapper(); - - /** - * Creates instance of the client. - * - * @param httpClient HTTP client - * @param url URL of the GPT REST end point - * @param cred credentials - * @param index index name - * @param collectionsFieldName collections field name - */ - public Client(CloseableHttpClient httpClient, URL url, SimpleCredentials cred, String index, String collectionsFieldName) { - this.httpClient = httpClient; - this.url = url; - this.cred = cred; - this.index = StringUtils.defaultIfBlank(index, DEFAULT_INDEX); - this.collectionsFieldName = collectionsFieldName; - - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - } - - /** - * Creates instance of the client. - * - * @param url URL of the GPT REST end point - * @param cred credentials - * @param index index name - * @param collectionsFieldName collections field name - */ - public Client(URL url, SimpleCredentials cred, String index, String collectionsFieldName) { - this(HttpClientBuilder.create().useSystemProperties().setRedirectStrategy(LaxRedirectStrategy.INSTANCE).build(), url, cred, index, collectionsFieldName); - } - - /** - * Publishes a document. - * - * @param data data to publish - * @param attributes extra attributes - * @param id custom id - * @param xml xml - * @param json json - * @param forceAdd true to force add. - * @param collections list of collections - * @return response information - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public PublishResponse publish( - PublishRequest data, - Map attributes, - String id, - String xml, String json, - boolean forceAdd, - String [] collections) throws IOException, URISyntaxException { - - ObjectNode jsonRequest = mapper.convertValue(data, ObjectNode.class); - if (xml != null) { - jsonRequest.put("xml", xml); - } - if (json != null) { - try { - ObjectNode jsonValue = mapper.readValue(json, ObjectNode.class); - jsonRequest.set("_json", jsonValue); - if (jsonValue.isObject()) { - Iterator> fldIter = jsonValue.fields(); - while (fldIter.hasNext()) { - Map.Entry fld = fldIter.next(); - - if (fld.getKey().equals("fullExtent")) { - JsonNode fullExtent = fld.getValue(); - - Double xmin = fullExtent.path("xmin").asDouble(); - Double ymin = fullExtent.path("ymin").asDouble(); - Double xmax = fullExtent.path("xmax").asDouble(); - Double ymax = fullExtent.path("ymax").asDouble(); - - if (attributes.containsKey(WKAConstants.WKA_BBOX)) { - Object boxObj = attributes.get(WKAConstants.WKA_BBOX); - if (boxObj!=null && boxObj instanceof Attribute) { - String parts [] = ((Attribute)boxObj).getValue().split(","); - if (parts!=null && parts.length==2) { - String ll[] = parts[0].split(" "); - String ur[] = parts[1].split(" "); - if (ll!=null && ll.length==2 && ur!=null && ur.length==2) { - Double b_xmin = parseDouble(ll[0]); - Double b_ymin = parseDouble(ll[1]); - Double b_xmax = parseDouble(ur[0]); - Double b_ymax = parseDouble(ur[1]); - - if (b_xmin!=null && b_ymin!=null && b_xmax!=null && b_ymax!=null) { - xmin = b_xmin; - ymin = b_ymin; - xmax = b_xmax; - ymax = b_ymax; - } + private static final Logger LOG = LoggerFactory.getLogger(Client.class); + private static final int BATCH_SIZE = 500; + + private static final String DEFAULT_INDEX = "metadata"; + private static final String REST_ITEM_URL = "rest/metadata/item"; + private static final String ELASTIC_SEARCH_URL = "elastic/{metadata}/_search"; + private static final String ELASTIC_SCROLL_URL = "elastic/_search/scroll"; + private static final String TOKEN_URL = "oauth/token"; + + private final CloseableHttpClient httpClient; + private final URL url; + private final SimpleCredentials cred; + private final String index; + private final String collectionsFieldName; + + private TokenInfo tokenInfo; + + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Creates instance of the client. + * + * @param httpClient HTTP client + * @param url URL of the GPT REST end point + * @param cred credentials + * @param index index name + * @param collectionsFieldName collections field name + */ + public Client(CloseableHttpClient httpClient, URL url, SimpleCredentials cred, String index, String collectionsFieldName) { + this.httpClient = httpClient; + this.url = url; + this.cred = cred; + this.index = StringUtils.defaultIfBlank(index, DEFAULT_INDEX); + this.collectionsFieldName = collectionsFieldName; + + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + /** + * Creates instance of the client. + * + * @param url URL of the GPT REST end point + * @param cred credentials + * @param index index name + * @param collectionsFieldName collections field name + */ + public Client(URL url, SimpleCredentials cred, String index, String collectionsFieldName) { + this(HttpClientBuilder.create().useSystemProperties().setRedirectStrategy(LaxRedirectStrategy.INSTANCE).build(), url, cred, index, collectionsFieldName); + } + + /** + * Publishes a document. + * + * @param data data to publish + * @param attributes extra attributes + * @param id custom id + * @param xml xml + * @param json json + * @param forceAdd true to force add. + * @param collections list of collections + * @return response information + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public PublishResponse publish( + PublishRequest data, + Map attributes, + String id, + String xml, String json, + boolean forceAdd, + String[] collections) throws IOException, URISyntaxException { + + ObjectNode jsonRequest = mapper.convertValue(data, ObjectNode.class); + if (xml != null) { + jsonRequest.put("xml", xml); + } + if (json != null) { + try { + ObjectNode jsonValue = mapper.readValue(json, ObjectNode.class); + jsonRequest.set("_json", jsonValue); + if (jsonValue.isObject()) { + Iterator> fldIter = jsonValue.fields(); + while (fldIter.hasNext()) { + Map.Entry fld = fldIter.next(); + + if (fld.getKey().equals("fullExtent")) { + JsonNode fullExtent = fld.getValue(); + + Double xmin = fullExtent.path("xmin").asDouble(); + Double ymin = fullExtent.path("ymin").asDouble(); + Double xmax = fullExtent.path("xmax").asDouble(); + Double ymax = fullExtent.path("ymax").asDouble(); + + if (attributes.containsKey(WKAConstants.WKA_BBOX)) { + Object boxObj = attributes.get(WKAConstants.WKA_BBOX); + if (boxObj != null && boxObj instanceof Attribute) { + String parts[] = ((Attribute) boxObj).getValue().split(","); + if (parts != null && parts.length == 2) { + String ll[] = parts[0].split(" "); + String ur[] = parts[1].split(" "); + if (ll != null && ll.length == 2 && ur != null && ur.length == 2) { + Double b_xmin = parseDouble(ll[0]); + Double b_ymin = parseDouble(ll[1]); + Double b_xmax = parseDouble(ur[0]); + Double b_ymax = parseDouble(ur[1]); + + if (b_xmin != null && b_ymin != null && b_xmax != null && b_ymax != null) { + xmin = b_xmin; + ymin = b_ymin; + xmax = b_xmax; + ymax = b_ymax; + } + } + } + } + } + + ObjectNode envelope_geo = mapper.createObjectNode(); + envelope_geo.put("type", "envelope"); + ArrayNode coordinates = mapper.createArrayNode(); + ArrayNode southWest = coordinates.addArray(); + southWest.add(Math.max(xmin, -180.0)); + southWest.add(Math.min(ymax, 90.0)); + ArrayNode northEast = coordinates.addArray(); + northEast.add(Math.min(xmax, 180.0)); + northEast.add(Math.max(ymin, -90.0)); + envelope_geo.set("coordinates", coordinates); + + jsonRequest.set("envelope_geo", envelope_geo); + + double lon = (xmin + xmax) / 2.0; + double lat = (ymin + ymax) / 2.0; + + ObjectNode envelope_cen_pt = mapper.createObjectNode(); + envelope_cen_pt.put("lon", lon); + envelope_cen_pt.put("lat", lat); + + jsonRequest.set("envelope_cen_pt", envelope_cen_pt); + } + + switch (fld.getValue().getNodeType()) { + case STRING: + String s_format = "%s_txt"; + switch (fld.getKey()) { + case "allowedUploadFileTypes": + case "capabilities": + case "configuredState": + case "clusterName": + case "executionType": + case "geometryType": + case "htmlPopupType": + case "isolationLevel": + case "loadBalancing": + case "supportedQueryFormats": + case "tags": + case "type": + case "typeName": + case "units": + s_format = "%s_s"; + break; + case "title": + case "description": + case "fileid": + s_format = "%s"; + break; + } + jsonRequest.put(String.format(s_format, fld.getKey()), fld.getValue().asText()); + break; + case NUMBER: + jsonRequest.put(String.format("%s_d", fld.getKey()), fld.getValue().asDouble()); + break; + case BOOLEAN: + jsonRequest.put(String.format("%s_b", fld.getKey()), fld.getValue().asBoolean()); + break; + case ARRAY: + jsonRequest.set(String.format("%s", fld.getKey()), fld.getValue()); + break; + case OBJECT: +// jsonRequest.set(String.format("%s_obj", fld.getKey()), fld.getValue()); + break; + } } - } } - } + } catch (Exception ex) { + LOG.debug(String.format("Invalid json received.", json), ex); + } + } + + for (Map.Entry entry : attributes.entrySet()) { + if (entry.getValue() == null) { + jsonRequest.putNull(entry.getKey()); + } else { + if (entry.getValue() instanceof String) { + jsonRequest.put(entry.getKey(), (String) entry.getValue()); + } else if (entry.getValue() instanceof Double) { + jsonRequest.put(entry.getKey(), (Double) entry.getValue()); + } else if (entry.getValue() instanceof BigDecimal) { + jsonRequest.put(entry.getKey(), ((BigDecimal) entry.getValue()).doubleValue()); + } else if (entry.getValue() instanceof Float) { + jsonRequest.put(entry.getKey(), (Float) entry.getValue()); + } else if (entry.getValue() instanceof Long) { + jsonRequest.put(entry.getKey(), (Long) entry.getValue()); + } else if (entry.getValue() instanceof BigInteger) { + jsonRequest.put(entry.getKey(), ((BigInteger) entry.getValue()).longValue()); + } else if (entry.getValue() instanceof Integer) { + jsonRequest.put(entry.getKey(), (Integer) entry.getValue()); + } else if (entry.getValue() instanceof Boolean) { + jsonRequest.put(entry.getKey(), (Boolean) entry.getValue()); + } else if (entry.getValue() instanceof JsonNode) { + jsonRequest.set(entry.getKey(), (JsonNode) entry.getValue()); + } + } + } - ObjectNode envelope_geo = mapper.createObjectNode(); - envelope_geo.put("type", "envelope"); - ArrayNode coordinates = mapper.createArrayNode(); - ArrayNode southWest = coordinates.addArray(); - southWest.add(Math.max(xmin, -180.0)); - southWest.add(Math.min(ymax, 90.0)); - ArrayNode northEast = coordinates.addArray(); - northEast.add(Math.min(xmax, 180.0)); - northEast.add(Math.max(ymin, -90.0)); - envelope_geo.set("coordinates", coordinates); + if (collections != null) { + List collectionsList = Arrays.stream(collections) + .map(StringUtils::trimToNull) + .filter(collection -> collection != null) + .collect(Collectors.toList()); - jsonRequest.set("envelope_geo", envelope_geo); + if (!collectionsList.isEmpty()) { + ArrayNode collectionsArray = jsonRequest.putArray(collectionsFieldName); + collectionsList.forEach(collectionsArray::add); + } + } - double lon = (xmin + xmax) / 2.0; - double lat = (ymin + ymax) / 2.0; + String strRequest = mapper.writeValueAsString(jsonRequest); + StringEntity entity = new StringEntity(strRequest, "UTF-8"); - ObjectNode envelope_cen_pt = mapper.createObjectNode(); - envelope_cen_pt.put("lon", lon); - envelope_cen_pt.put("lat", lat); + List ids = !forceAdd ? queryIds("src_uri_s", data.src_uri_s, 1) : Collections.emptyList(); - jsonRequest.set("envelope_cen_pt", envelope_cen_pt); + URI pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); + try { + return publish(pubUri, entity, data.sys_owner_s); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); + return publish(pubUri, entity, data.sys_owner_s); + } else { + throw ex; } + } + } - switch (fld.getValue().getNodeType()) { - case STRING: - String s_format = "%s_txt"; - switch (fld.getKey()) { - case "allowedUploadFileTypes": - case "capabilities": - case "configuredState": - case "clusterName": - case "executionType": - case "geometryType": - case "htmlPopupType": - case "isolationLevel": - case "loadBalancing": - case "supportedQueryFormats": - case "tags": - case "type": - case "typeName": - case "units": - s_format = "%s_s"; - break; - case "title": - case "description": - case "fileid": - s_format = "%s"; - break; + private Double parseDouble(String val) { + try { + return Double.parseDouble(val); + } catch (Exception ex) { + return null; + } + } + + /** + * Reads metadata. + * + * @param id id of the metadata + * @return string representing metadata + * @throws URISyntaxException if invalid URI + * @throws IOException if reading metadata fails + */ + public String readXml(String id) throws URISyntaxException, IOException { + URI xmlUri = createXmlUri(id); + try { + return readContent(xmlUri); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + xmlUri = createXmlUri(id); + return readContent(xmlUri); + } else { + throw ex; + } + } + } + + /** + * Reads metadata. + * + * @param id id of the metadata + * @return string representing metadata + * @throws URISyntaxException if invalid URI + * @throws IOException if reading metadata fails + */ + public String readJson(String id) throws URISyntaxException, IOException { + URI jsonUri = createJsonUri(id); + try { + String content = readContent(jsonUri); + try { + JsonNode root = mapper.readTree(content); + JsonNode json = null; + if (root.isObject() && root.has("_source") && root.get("_source").isObject() && root.get("_source").has("_json")) { + json = root.get("_source").get("_json"); + return mapper.writeValueAsString(json); + } //if _source._json not there then take _source + else if (root.isObject() && root.has("_source") && json == null) { + json = root.get("_source"); + return mapper.writeValueAsString(json); } - jsonRequest.put(String.format(s_format, fld.getKey()), fld.getValue().asText()); - break; - case NUMBER: - jsonRequest.put(String.format("%s_d", fld.getKey()), fld.getValue().asDouble()); - break; - case BOOLEAN: - jsonRequest.put(String.format("%s_b", fld.getKey()), fld.getValue().asBoolean()); - break; - case ARRAY: - jsonRequest.set(String.format("%s", fld.getKey()), fld.getValue()); - break; - case OBJECT: -// jsonRequest.set(String.format("%s_obj", fld.getKey()), fld.getValue()); - break; + } catch (IOException ex) { + // ignore + } + return null; + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + jsonUri = createJsonUri(id); + return readContent(jsonUri); + } else { + throw ex; } - } } - } catch (Exception ex) { - LOG.debug(String.format("Invalid json received.", json), ex); - } - } - - for (Map.Entry entry : attributes.entrySet()) { - if (entry.getValue() == null) { - jsonRequest.putNull(entry.getKey()); - } else { - if (entry.getValue() instanceof String) { - jsonRequest.put(entry.getKey(), (String) entry.getValue()); - } else if (entry.getValue() instanceof Double) { - jsonRequest.put(entry.getKey(), (Double) entry.getValue()); - } else if (entry.getValue() instanceof BigDecimal) { - jsonRequest.put(entry.getKey(), ((BigDecimal) entry.getValue()).doubleValue()); - } else if (entry.getValue() instanceof Float) { - jsonRequest.put(entry.getKey(), (Float) entry.getValue()); - } else if (entry.getValue() instanceof Long) { - jsonRequest.put(entry.getKey(), (Long) entry.getValue()); - } else if (entry.getValue() instanceof BigInteger) { - jsonRequest.put(entry.getKey(), ((BigInteger) entry.getValue()).longValue()); - } else if (entry.getValue() instanceof Integer) { - jsonRequest.put(entry.getKey(), (Integer) entry.getValue()); - } else if (entry.getValue() instanceof Boolean) { - jsonRequest.put(entry.getKey(), (Boolean) entry.getValue()); - } else if (entry.getValue() instanceof JsonNode) { - jsonRequest.set(entry.getKey(), (JsonNode) entry.getValue()); + } + + /** + * Reads metadata. + * + * @param id id of the metadata + * @return string representing metadata + * @throws URISyntaxException if invalid URI + * @throws IOException if reading metadata fails + */ + public EntryRef readItem(String id) throws URISyntaxException, IOException { + URI itemUri = createItemUri(id); + try { + return readItem(itemUri); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + itemUri = createItemUri(id); + return readItem(itemUri); + } else { + throw ex; + } } - } - } - - if (collections!=null) { - List collectionsList = Arrays.stream(collections) - .map(StringUtils::trimToNull) - .filter(collection -> collection!=null) - .collect(Collectors.toList()); - - if (!collectionsList.isEmpty()) { - ArrayNode collectionsArray = jsonRequest.putArray(collectionsFieldName); - collectionsList.forEach(collectionsArray::add); - } - } - - String strRequest = mapper.writeValueAsString(jsonRequest); - StringEntity entity = new StringEntity(strRequest, "UTF-8"); - - List ids = !forceAdd ? queryIds("src_uri_s", data.src_uri_s, 1) : Collections.emptyList(); - - URI pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); - try { - return publish(pubUri, entity, data.sys_owner_s); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); - return publish(pubUri, entity, data.sys_owner_s); - } else { - throw ex; - } - } - } - - private Double parseDouble(String val) { - try { - return Double.parseDouble(val); - } catch (Exception ex) { - return null; - } - } - - /** - * Reads metadata. - * - * @param id id of the metadata - * @return string representing metadata - * @throws URISyntaxException if invalid URI - * @throws IOException if reading metadata fails - */ - public String readXml(String id) throws URISyntaxException, IOException { - URI xmlUri = createXmlUri(id); - try { - return readContent(xmlUri); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - xmlUri = createXmlUri(id); - return readContent(xmlUri); - } else { - throw ex; - } - } - } - - /** - * Reads metadata. - * - * @param id id of the metadata - * @return string representing metadata - * @throws URISyntaxException if invalid URI - * @throws IOException if reading metadata fails - */ - public String readJson(String id) throws URISyntaxException, IOException { - URI jsonUri = createJsonUri(id); - try { - String content = readContent(jsonUri); - try { - JsonNode root = mapper.readTree(content); - if (root.isObject() && root.has("_source") && root.get("_source").isObject() && root.get("_source").has("_json")) { - JsonNode json = root.get("_source").get("_json"); - return mapper.writeValueAsString(json); + } + + /** + * Returns listIds of ids. + * + * @return listIds of ids or null if no more ids. + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public List listIds() throws URISyntaxException, IOException { + return queryIds(null, null, BATCH_SIZE); + } + + @Override + public void close() throws IOException { + if (httpClient instanceof Closeable) { + ((Closeable) httpClient).close(); } - } catch (IOException ex) { - // ignore - } - return null; - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - jsonUri = createJsonUri(id); - return readContent(jsonUri); - } else { - throw ex; - } - } - } - - /** - * Reads metadata. - * - * @param id id of the metadata - * @return string representing metadata - * @throws URISyntaxException if invalid URI - * @throws IOException if reading metadata fails - */ - public EntryRef readItem(String id) throws URISyntaxException, IOException { - URI itemUri = createItemUri(id); - try { - return readItem(itemUri); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - itemUri = createItemUri(id); - return readItem(itemUri); - } else { - throw ex; - } - } - } - - /** - * Returns listIds of ids. - * - * @return listIds of ids or null if no more ids. - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public List listIds() throws URISyntaxException, IOException { - return queryIds(null, null, BATCH_SIZE); - } - - @Override - public void close() throws IOException { - if (httpClient instanceof Closeable) { - ((Closeable) httpClient).close(); - } - } - - /** - * Query items by src_source_uri_s. - * - * @param src_source_uri_s query - * @return query response - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public List queryBySource(String src_source_uri_s) throws IOException, URISyntaxException { - return queryIds("src_source_uri_s", src_source_uri_s, BATCH_SIZE); - } - - /** - * Deletes record by id. - * - * @param id record id - * @return publish response - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public PublishResponse delete(String id) throws URISyntaxException, IOException { - URI deleteUri = createItemUri(id); - try { - return delete(deleteUri); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - deleteUri = createItemUri(id); - return delete(deleteUri); - } else { - throw ex; - } - } - } - - private PublishResponse publish(URI uri, StringEntity entity, String owner) throws IOException, URISyntaxException { - HttpPut put = new HttpPut(uri); - put.setConfig(DEFAULT_REQUEST_CONFIG); - put.setEntity(entity); - put.setHeader("Content-Type", "application/json; charset=UTF-8"); - put.setHeader("User-Agent", HttpConstants.getUserAgent()); - - PublishResponse response = execute(put, PublishResponse.class); - if (response.getError() == null && owner != null) { - changeOwner(response.getId(), owner); - } - - return response; - } - - private EntryRef readItem(URI uri) throws URISyntaxException, IOException { - HttpGet get = new HttpGet(uri); - get.setConfig(DEFAULT_REQUEST_CONFIG); - get.setHeader("User-Agent", HttpConstants.getUserAgent()); - Hit hit = execute(get, Hit.class); - return new EntryRef(hit._id, readUri(hit._source, uri), readLastUpdated(hit._source, new Date())); - } - - private String readContent(URI uri) throws URISyntaxException, IOException { - HttpGet get = new HttpGet(uri); - get.setConfig(DEFAULT_REQUEST_CONFIG); - get.setHeader("User-Agent", HttpConstants.getUserAgent()); - - try (CloseableHttpResponse httpResponse = httpClient.execute(get); InputStream contentStream = httpResponse.getEntity().getContent();) { - if (httpResponse.getStatusLine().getStatusCode() >= 400) { - throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); - } - String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); - String responseContent = IOUtils.toString(contentStream, "UTF-8"); - LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); - return responseContent; - } - } - - private PublishResponse changeOwner(String id, String owner) throws IOException, URISyntaxException { - URI uri = createChangeOwnerUri(id, owner); - HttpPut put = new HttpPut(uri); - put.setConfig(DEFAULT_REQUEST_CONFIG); - put.setHeader("Content-Type", "application/json; charset=UTF-8"); - put.setHeader("User-Agent", HttpConstants.getUserAgent()); - - return execute(put, PublishResponse.class); - } - - private URI readUri(QueryResponse.Source source, URI defUri) { - if (source != null && source.src_uri_s != null) { - try { - return new URI(source.src_uri_s); - } catch (Exception ex) { - } - } - return defUri; - } - - private Date readLastUpdated(QueryResponse.Source source, Date defDate) { - if (source != null && source.src_lastupdate_dt != null) { - try { - return Date.from(ZonedDateTime.from(DateTimeFormatter.ISO_DATE_TIME.parse(source.src_lastupdate_dt)).toInstant()); - } catch (Exception ex) { - } - } - return defDate; - } - - private PublishResponse delete(URI uri) throws URISyntaxException, IOException { - HttpDelete del = new HttpDelete(uri); - del.setConfig(DEFAULT_REQUEST_CONFIG); - del.setHeader("User-Agent", HttpConstants.getUserAgent()); - return execute(del, PublishResponse.class); - } - - private URI createItemsUri() throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL)); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - private URI createItemUri(String id) throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - private URI createChangeOwnerUri(String id, String owner) throws URISyntaxException, IOException { - return new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/owner/" + owner)) - .addParameter("access_token", getAccessToken()) - .build(); - } - - private URI createXmlUri(String id) throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/xml")); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - private URI createJsonUri(String id) throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - /** - * Query ids. - * - * @param term term to query - * @param value value of the term - * @param batchSize batch size (note: size 1 indicates looking for the first - * only) - * @return listIds of ids - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - private List queryIds(String term, String value, long batchSize) throws IOException, URISyntaxException { - Set ids = new HashSet<>(); - String search_after = null; - - ObjectNode root = mapper.createObjectNode(); - root.put("size", batchSize); - root.set("_source", mapper.createArrayNode().add("_id")); - root.set("sort", mapper.createArrayNode().add(mapper.createObjectNode().put("_id", "asc"))); - if (term != null && value != null) { - root.set("query", mapper.createObjectNode().set("match", mapper.createObjectNode().put(term, value))); - } - - do { - URIBuilder builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())); - if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); - } - - if (search_after != null) { - root.set("search_after", mapper.createArrayNode().add(search_after)); - } - - String json = mapper.writeValueAsString(root); - HttpEntity entity = new StringEntity(json, ContentType.APPLICATION_JSON); - - QueryResponse response = query(builder, entity); - if (response!=null && response.status!=null && response.status==400 && search_after==null) { - // This indicates it could be an old version of Elastic Search behind the Geoportal. - // Fall back to using scroll API - return queryIdsScroll(term, value, batchSize); - } - - search_after = null; - if (response != null && response.hasHits()) { - List responseIds = response.hits.hits.stream().map(hit -> hit._id).collect(Collectors.toList()); - ids.addAll(responseIds); - - // if argument 'size' is 1 that means looking for the first one only; otherwise looking for every possible - search_after = batchSize > 1 ? responseIds.get(responseIds.size() - 1) : null; - } - } while (search_after != null && !Thread.currentThread().isInterrupted()); - - return ids.stream().collect(Collectors.toList()); - } - - private List queryIdsScroll(String term, String value, long size) throws IOException, URISyntaxException { - ArrayList ids = new ArrayList<>(); - SearchContext searchContext = new SearchContext(); - - while (!Thread.currentThread().isInterrupted()) { - QueryResponse response = query(term, value, size, searchContext); - if (Thread.currentThread().isInterrupted()) { - break; - } - if (response.hits == null || response.hits.hits == null || response.hits.hits.isEmpty()) { - break; - } - ids.addAll(response.hits.hits.stream() - .map(h -> h._id) - .filter(id -> id != null) - .collect(Collectors.toList())); - } - - return ids; - } - - private void clearToken() { - tokenInfo = null; - } - - private QueryResponse query(String term, String value, long size, SearchContext searchContext) throws IOException, URISyntaxException { - URI uri = createQueryUri(searchContext); - HttpEntity httpEntity = createQueryEntity(term, value, size, searchContext); - try { - QueryResponse response = query(uri, httpEntity); - searchContext._scroll_id = response._scroll_id; - return response; - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - uri = createQueryUri(searchContext); - httpEntity = createQueryEntity(term, value, size, searchContext); - QueryResponse response = query(uri, httpEntity); - searchContext._scroll_id = response._scroll_id; + } + + /** + * Query items by src_source_uri_s. + * + * @param src_source_uri_s query + * @return query response + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public List queryBySource(String src_source_uri_s) throws IOException, URISyntaxException { + return queryIds("src_source_uri_s", src_source_uri_s, BATCH_SIZE); + } + + /** + * Deletes record by id. + * + * @param id record id + * @return publish response + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public PublishResponse delete(String id) throws URISyntaxException, IOException { + URI deleteUri = createItemUri(id); + try { + return delete(deleteUri); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + deleteUri = createItemUri(id); + return delete(deleteUri); + } else { + throw ex; + } + } + } + + private PublishResponse publish(URI uri, StringEntity entity, String owner) throws IOException, URISyntaxException { + HttpPut put = new HttpPut(uri); + put.setConfig(DEFAULT_REQUEST_CONFIG); + put.setEntity(entity); + put.setHeader("Content-Type", "application/json; charset=UTF-8"); + put.setHeader("User-Agent", HttpConstants.getUserAgent()); + + PublishResponse response = execute(put, PublishResponse.class); + if (response.getError() == null && owner != null) { + changeOwner(response.getId(), owner); + } + return response; - } else { - throw ex; - } } - } - private QueryResponse query(URIBuilder builder, HttpEntity entity) throws IOException, URISyntaxException { - if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); + private EntryRef readItem(URI uri) throws URISyntaxException, IOException { + HttpGet get = new HttpGet(uri); + get.setConfig(DEFAULT_REQUEST_CONFIG); + get.setHeader("User-Agent", HttpConstants.getUserAgent()); + Hit hit = execute(get, Hit.class); + return new EntryRef(hit._id, readUri(hit._source, uri), readLastUpdated(hit._source, new Date())); + } + + private String readContent(URI uri) throws URISyntaxException, IOException { + HttpGet get = new HttpGet(uri); + get.setConfig(DEFAULT_REQUEST_CONFIG); + get.setHeader("User-Agent", HttpConstants.getUserAgent()); + + try ( CloseableHttpResponse httpResponse = httpClient.execute(get); InputStream contentStream = httpResponse.getEntity().getContent();) { + if (httpResponse.getStatusLine().getStatusCode() >= 400) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); + String responseContent = IOUtils.toString(contentStream, "UTF-8"); + LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); + return responseContent; + } + } + + private PublishResponse changeOwner(String id, String owner) throws IOException, URISyntaxException { + URI uri = createChangeOwnerUri(id, owner); + HttpPut put = new HttpPut(uri); + put.setConfig(DEFAULT_REQUEST_CONFIG); + put.setHeader("Content-Type", "application/json; charset=UTF-8"); + put.setHeader("User-Agent", HttpConstants.getUserAgent()); + + return execute(put, PublishResponse.class); + } + + private URI readUri(QueryResponse.Source source, URI defUri) { + if (source != null && source.src_uri_s != null) { + try { + return new URI(source.src_uri_s); + } catch (Exception ex) { + } + } + return defUri; } - QueryResponse response = null; - try { - response = query(builder.build(), entity); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); + private Date readLastUpdated(QueryResponse.Source source, Date defDate) { + if (source != null && source.src_lastupdate_dt != null) { + try { + return Date.from(ZonedDateTime.from(DateTimeFormatter.ISO_DATE_TIME.parse(source.src_lastupdate_dt)).toInstant()); + } catch (Exception ex) { + } + } + return defDate; + } + + private PublishResponse delete(URI uri) throws URISyntaxException, IOException { + HttpDelete del = new HttpDelete(uri); + del.setConfig(DEFAULT_REQUEST_CONFIG); + del.setHeader("User-Agent", HttpConstants.getUserAgent()); + return execute(del, PublishResponse.class); + } + + private URI createItemsUri() throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL)); if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); + b.addParameter("access_token", getAccessToken()); } - response = query(builder.build(), entity); - } else { - throw ex; - } + return b.build(); } - return response; - } + private URI createItemUri(String id) throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); + if (cred != null && !cred.isEmpty()) { + b.addParameter("access_token", getAccessToken()); + } + return b.build(); + } - private QueryResponse query(URI uri, HttpEntity httpEntity) throws IOException, URISyntaxException { - HttpPost request = new HttpPost(uri); - request.setEntity(httpEntity); + private URI createChangeOwnerUri(String id, String owner) throws URISyntaxException, IOException { + return new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/owner/" + owner)) + .addParameter("access_token", getAccessToken()) + .build(); + } - request.setConfig(DEFAULT_REQUEST_CONFIG); - request.setHeader("Content-Type", "application/json"); - request.setHeader("User-Agent", HttpConstants.getUserAgent()); + private URI createXmlUri(String id) throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/xml")); + if (cred != null && !cred.isEmpty()) { + b.addParameter("access_token", getAccessToken()); + } + return b.build(); + } + + private URI createJsonUri(String id) throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); + if (cred != null && !cred.isEmpty()) { + b.addParameter("access_token", getAccessToken()); + } + return b.build(); + } - return execute(request, QueryResponse.class); - } + /** + * Query ids. + * + * @param term term to query + * @param value value of the term + * @param batchSize batch size (note: size 1 indicates looking for the first + * only) + * @return listIds of ids + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + private List queryIds(String term, String value, long batchSize) throws IOException, URISyntaxException { + Set ids = new HashSet<>(); + String search_after = null; + + ObjectNode root = mapper.createObjectNode(); + root.put("size", batchSize); + root.set("_source", mapper.createArrayNode().add("_id")); + root.set("sort", mapper.createArrayNode().add(mapper.createObjectNode().put("_id", "asc"))); + if (term != null && value != null) { + root.set("query", mapper.createObjectNode().set("match", mapper.createObjectNode().put(term, value))); + } - private String createElasticSearchUrl() { - return ELASTIC_SEARCH_URL.replaceAll("\\{metadata\\}", index); - } + do { + URIBuilder builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())); + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } - private HttpEntity createQueryEntity(String term, String value, long size, SearchContext searchContext) { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode node = mapper.createObjectNode(); - if (searchContext._scroll_id == null) { - node.put("size", size); - if (term != null && value != null) { - ObjectNode query = mapper.createObjectNode(); - node.set("query", query); + if (search_after != null) { + root.set("search_after", mapper.createArrayNode().add(search_after)); + } - ObjectNode match = mapper.createObjectNode(); - query.set("match", match); + String json = mapper.writeValueAsString(root); + HttpEntity entity = new StringEntity(json, ContentType.APPLICATION_JSON); - match.put(term, value); - } - } else { - node.put("scroll", "1m"); - node.put("scroll_id", searchContext._scroll_id); + QueryResponse response = query(builder, entity); + if (response != null && response.status != null && response.status == 400 && search_after == null) { + // This indicates it could be an old version of Elastic Search behind the Geoportal. + // Fall back to using scroll API + return queryIdsScroll(term, value, batchSize); + } + + search_after = null; + if (response != null && response.hasHits()) { + List responseIds = response.hits.hits.stream().map(hit -> hit._id).collect(Collectors.toList()); + ids.addAll(responseIds); + + // if argument 'size' is 1 that means looking for the first one only; otherwise looking for every possible + search_after = batchSize > 1 ? responseIds.get(responseIds.size() - 1) : null; + } + } while (search_after != null && !Thread.currentThread().isInterrupted()); + + return ids.stream().collect(Collectors.toList()); } - return new StringEntity(node.toString(), ContentType.APPLICATION_JSON); - } + private List queryIdsScroll(String term, String value, long size) throws IOException, URISyntaxException { + ArrayList ids = new ArrayList<>(); + SearchContext searchContext = new SearchContext(); - private URI createQueryUri(SearchContext searchContext) throws IOException, URISyntaxException { - URIBuilder builder; + while (!Thread.currentThread().isInterrupted()) { + QueryResponse response = query(term, value, size, searchContext); + if (Thread.currentThread().isInterrupted()) { + break; + } + if (response.hits == null || response.hits.hits == null || response.hits.hits.isEmpty()) { + break; + } + ids.addAll(response.hits.hits.stream() + .map(h -> h._id) + .filter(id -> id != null) + .collect(Collectors.toList())); + } - if (searchContext._scroll_id == null) { - builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())) - .addParameter("scroll", "1m"); - } else { - builder = new URIBuilder(url.toURI().resolve(ELASTIC_SCROLL_URL)) - .addParameter("scroll_id", searchContext._scroll_id) - .addParameter("scroll", "1m"); + return ids; } - if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); + private void clearToken() { + tokenInfo = null; } - return builder.build(); - } + private QueryResponse query(String term, String value, long size, SearchContext searchContext) throws IOException, URISyntaxException { + URI uri = createQueryUri(searchContext); + HttpEntity httpEntity = createQueryEntity(term, value, size, searchContext); + try { + QueryResponse response = query(uri, httpEntity); + searchContext._scroll_id = response._scroll_id; + return response; + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + uri = createQueryUri(searchContext); + httpEntity = createQueryEntity(term, value, size, searchContext); + QueryResponse response = query(uri, httpEntity); + searchContext._scroll_id = response._scroll_id; + return response; + } else { + throw ex; + } + } + } - private T execute(HttpUriRequest req, Class clazz) throws IOException, URISyntaxException { - try (CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { - String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); - String responseContent = IOUtils.toString(contentStream, "UTF-8"); - LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); + private QueryResponse query(URIBuilder builder, HttpEntity entity) throws IOException, URISyntaxException { + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } - if (httpResponse.getStatusLine().getStatusCode() >= 400) { - T value = null; + QueryResponse response = null; try { - value = mapper.readValue(responseContent, clazz); - } catch (Exception ex) { - throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + response = query(builder.build(), entity); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } + response = query(builder.build(), entity); + } else { + throw ex; + } + } + + return response; + } + + private QueryResponse query(URI uri, HttpEntity httpEntity) throws IOException, URISyntaxException { + HttpPost request = new HttpPost(uri); + request.setEntity(httpEntity); + + request.setConfig(DEFAULT_REQUEST_CONFIG); + request.setHeader("Content-Type", "application/json"); + request.setHeader("User-Agent", HttpConstants.getUserAgent()); + + return execute(request, QueryResponse.class); + } + + private String createElasticSearchUrl() { + return ELASTIC_SEARCH_URL.replaceAll("\\{metadata\\}", index); + } + + private HttpEntity createQueryEntity(String term, String value, long size, SearchContext searchContext) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode node = mapper.createObjectNode(); + if (searchContext._scroll_id == null) { + node.put("size", size); + if (term != null && value != null) { + ObjectNode query = mapper.createObjectNode(); + node.set("query", query); + + ObjectNode match = mapper.createObjectNode(); + query.set("match", match); + + match.put(term, value); + } + } else { + node.put("scroll", "1m"); + node.put("scroll_id", searchContext._scroll_id); + } + + return new StringEntity(node.toString(), ContentType.APPLICATION_JSON); + } + + private URI createQueryUri(SearchContext searchContext) throws IOException, URISyntaxException { + URIBuilder builder; + + if (searchContext._scroll_id == null) { + builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())) + .addParameter("scroll", "1m"); + } else { + builder = new URIBuilder(url.toURI().resolve(ELASTIC_SCROLL_URL)) + .addParameter("scroll_id", searchContext._scroll_id) + .addParameter("scroll", "1m"); + } + + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } + + return builder.build(); + } + + private T execute(HttpUriRequest req, Class clazz) throws IOException, URISyntaxException { + try ( CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { + String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); + String responseContent = IOUtils.toString(contentStream, "UTF-8"); + LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); + + if (httpResponse.getStatusLine().getStatusCode() >= 400) { + T value = null; + try { + value = mapper.readValue(responseContent, clazz); + } catch (Exception ex) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + if (value == null) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + return value; + } + + return mapper.readValue(responseContent, clazz); + } + } + + private String getAccessToken() throws URISyntaxException, IOException { + LocalDateTime now = LocalDateTime.now(); + if (tokenInfo == null || tokenInfo.validTill.minusSeconds(60).isBefore(now)) { + Token token = generateToken(); + if (token.access_token == null) { + throw new IOException("Error obtaining access token"); + } + TokenInfo ti = new TokenInfo(); + ti.token = token; + ti.validTill = now.plusSeconds(token.expires_in); + tokenInfo = ti; } - if (value == null) { - throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + return tokenInfo.token.access_token; + } + + private Token generateToken() throws URISyntaxException, UnsupportedEncodingException, IOException { + HttpPost post = new HttpPost(url.toURI().resolve(TOKEN_URL)); + post.setConfig(DEFAULT_REQUEST_CONFIG); + post.setHeader("User-Agent", HttpConstants.getUserAgent()); + post.setHeader("Content-Type", "application/x-www-form-urlencoded"); + post.setHeader("Accept", "application/json"); + HashMap params = new HashMap<>(); + if (cred != null) { + params.put("username", StringUtils.trimToEmpty(cred.getUserName())); + params.put("password", StringUtils.trimToEmpty(cred.getPassword())); } - return value; - } - - return mapper.readValue(responseContent, clazz); - } - } - - private String getAccessToken() throws URISyntaxException, IOException { - LocalDateTime now = LocalDateTime.now(); - if (tokenInfo == null || tokenInfo.validTill.minusSeconds(60).isBefore(now)) { - Token token = generateToken(); - if (token.access_token == null) { - throw new IOException("Error obtaining access token"); - } - TokenInfo ti = new TokenInfo(); - ti.token = token; - ti.validTill = now.plusSeconds(token.expires_in); - tokenInfo = ti; - } - return tokenInfo.token.access_token; - } - - private Token generateToken() throws URISyntaxException, UnsupportedEncodingException, IOException { - HttpPost post = new HttpPost(url.toURI().resolve(TOKEN_URL)); - post.setConfig(DEFAULT_REQUEST_CONFIG); - post.setHeader("User-Agent", HttpConstants.getUserAgent()); - post.setHeader("Content-Type", "application/x-www-form-urlencoded"); - post.setHeader("Accept", "application/json"); - HashMap params = new HashMap<>(); - if (cred != null) { - params.put("username", StringUtils.trimToEmpty(cred.getUserName())); - params.put("password", StringUtils.trimToEmpty(cred.getPassword())); - } - params.put("grant_type", "password"); - params.put("client_id", "geoportal-client"); - HttpEntity entity = new UrlEncodedFormEntity(params.entrySet().stream() - .map(e -> new BasicNameValuePair(e.getKey(), e.getValue())).collect(Collectors.toList())); - post.setEntity(entity); - - return execute(post, Token.class); - } - - /** - * Search context. - */ - public static class SearchContext { - - public String _scroll_id; - } - - /** - * Access token. - */ - public static class Token { - - public String access_token; - public String token_type; - public Long expires_in; - public String scope; - public String jti; - } - - private static class TokenInfo { - - public Token token; - public LocalDateTime validTill; - } + params.put("grant_type", "password"); + params.put("client_id", "geoportal-client"); + HttpEntity entity = new UrlEncodedFormEntity(params.entrySet().stream() + .map(e -> new BasicNameValuePair(e.getKey(), e.getValue())).collect(Collectors.toList())); + post.setEntity(entity); + + return execute(post, Token.class); + } + + /** + * Search context. + */ + public static class SearchContext { + + public String _scroll_id; + } + + /** + * Access token. + */ + public static class Token { + + public String access_token; + public String token_type; + public Long expires_in; + public String scope; + public String jti; + } + + private static class TokenInfo { + + public Token token; + public LocalDateTime validTill; + } } diff --git a/geoportal-commons/geoportal-commons-meta/pom.xml b/geoportal-commons/geoportal-commons-meta/pom.xml index c3a801245..89e7e7f78 100644 --- a/geoportal-commons/geoportal-commons-meta/pom.xml +++ b/geoportal-commons/geoportal-commons-meta/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-meta Esri :: Geoportal Server :: Commons :: Meta diff --git a/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java b/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java index e76a98755..5bbafd2f8 100644 --- a/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java +++ b/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java @@ -33,10 +33,12 @@ public final class WKAConstants { public static final String WKA_BBOX = "bbox"; public static final String WKA_MODIFIED = "modified"; public static final String WKA_REFERENCES = "references"; + public static final String WKA_METADATA_XML = "metadataXML"; private static final Set all = new HashSet(Arrays.asList(new String[]{ WKA_IDENTIFIER, WKA_TITLE, WKA_DESCRIPTION, WKA_RESOURCE_URL, - WKA_RESOURCE_URL_SCHEME, WKA_BBOX, WKA_THUMBNAIL_URL, WKA_MODIFIED, WKA_REFERENCES + WKA_RESOURCE_URL_SCHEME, WKA_BBOX, WKA_THUMBNAIL_URL, WKA_MODIFIED, WKA_REFERENCES, + WKA_METADATA_XML })); /** diff --git a/geoportal-commons/geoportal-commons-oai-client/pom.xml b/geoportal-commons/geoportal-commons-oai-client/pom.xml index 248d7189a..887b3627d 100644 --- a/geoportal-commons/geoportal-commons-oai-client/pom.xml +++ b/geoportal-commons/geoportal-commons-oai-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-oai-client Esri :: Geoportal Server :: Commons :: OAI-PMH Client diff --git a/geoportal-commons/geoportal-commons-pdf/pom.xml b/geoportal-commons/geoportal-commons-pdf/pom.xml index 65eb222ff..13f631052 100644 --- a/geoportal-commons/geoportal-commons-pdf/pom.xml +++ b/geoportal-commons/geoportal-commons-pdf/pom.xml @@ -6,7 +6,7 @@ geoportal-commons com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-commons-pdf jar diff --git a/geoportal-commons/geoportal-commons-robots/pom.xml b/geoportal-commons/geoportal-commons-robots/pom.xml index 80bb09f2c..d60714707 100644 --- a/geoportal-commons/geoportal-commons-robots/pom.xml +++ b/geoportal-commons/geoportal-commons-robots/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 commons-robots Esri :: Geoportal Server :: Commons :: Robots diff --git a/geoportal-commons/geoportal-commons-stac-client/pom.xml b/geoportal-commons/geoportal-commons-stac-client/pom.xml index c5aa48ec4..208cb9f2c 100644 --- a/geoportal-commons/geoportal-commons-stac-client/pom.xml +++ b/geoportal-commons/geoportal-commons-stac-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-stac-client jar diff --git a/geoportal-commons/geoportal-commons-thredds-client/pom.xml b/geoportal-commons/geoportal-commons-thredds-client/pom.xml index 7df822c27..381e496fb 100644 --- a/geoportal-commons/geoportal-commons-thredds-client/pom.xml +++ b/geoportal-commons/geoportal-commons-thredds-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-thredds-client Esri :: Geoportal Server :: Commons :: THREDDS Client diff --git a/geoportal-commons/geoportal-commons-utils/pom.xml b/geoportal-commons/geoportal-commons-utils/pom.xml index ba6a9ff0f..5b300eca2 100644 --- a/geoportal-commons/geoportal-commons-utils/pom.xml +++ b/geoportal-commons/geoportal-commons-utils/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 commons-utils jar diff --git a/geoportal-commons/pom.xml b/geoportal-commons/pom.xml index 3b9ee04a6..a3e726d2d 100644 --- a/geoportal-commons/pom.xml +++ b/geoportal-commons/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-commons pom diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml b/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml index 1dad8b223..5ac6223bf 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-agp-publisher Esri :: Geoportal Server :: Harvester :: Data Publisher :: ArcGIS Portal @@ -14,7 +14,7 @@ ${project.groupId} geoportal-commons-agp-client - 2.7.0 + 2.7.1 jar diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java index dc5beb09a..1ce87676d 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java @@ -25,4 +25,6 @@ public static final String P_MAX_REDIRECTS = "agp-max-redirects"; public static final String P_UPLOAD = "agp-upload"; public static final String P_MARKDOWN2HTML = "agp-markdown2html"; + public static final String P_OAUTH = "agp-oauth"; + public static final String P_TOKEN = "agp-token"; } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 1ff49acb7..955630cd1 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -21,8 +21,8 @@ import com.esri.geoportal.commons.agp.client.FolderEntry; import com.esri.geoportal.commons.agp.client.ItemEntry; import com.esri.geoportal.commons.agp.client.ItemResponse; -import com.esri.geoportal.commons.constants.ItemType; import com.esri.geoportal.commons.agp.client.QueryResponse; +import com.esri.geoportal.commons.constants.ItemType; import com.esri.geoportal.commons.constants.MimeType; import com.esri.geoportal.commons.doc.DocUtils; import com.esri.geoportal.commons.meta.ArrayAttribute; @@ -48,7 +48,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.StringReader; -// import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -76,13 +75,8 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; -//import org.apache.http.HttpEntity; -//import org.apache.http.client.HttpResponseException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -//import org.apache.http.client.methods.HttpPost; -//import org.apache.http.client.utils.URIBuilder; -//import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.LaxRedirectStrategy; @@ -187,10 +181,26 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { //Node titleNode = markdownParser.parse(title); //title = htmlRenderer.render(titleNode); Node descriptionNode = markdownParser.parse(description); + description = htmlRenderer.render(descriptionNode); + description = description.replaceAll("[*] ","
  • "); + description = description.replaceAll("\\\\n","
    "); + description = replaceSpecialCharacters(description); + //description = description.replaceAll("[^\\x00-\\x7F]", ""); + + // also update arcgisMetadata with the new description as idAbs + // only if there is exactly 1 idAbs element + String[] metadataPieces = arcgisMetadata.split("idAbs>"); + if (metadataPieces.length == 3) { + metadataPieces[1] = "idAbs>"; + arcgisMetadata = metadataPieces[0] + metadataPieces[1] + metadataPieces[2]; + } } String sThumbnailUrl = StringUtils.trimToNull(getAttributeValue(attributes, WKAConstants.WKA_THUMBNAIL_URL, null)); String resourceUrl = getAttributeValue(attributes, WKAConstants.WKA_RESOURCE_URL, null); + + //Hardcoded url just for dev testing + //resourceUrl="https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/services/group_of_layers/FeatureServer/1"; // clean up resource URL resourceUrl = resourceUrl.replace("http:", "https:") .replace(":80/", "/"); @@ -272,86 +282,56 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { try { - // generate token - if (token == null) { - token = generateToken(); + // generate token + if ((definition.useOAuth()) || (token == null)) { + token = definition.getOAuthToken(); + //oAuth token was not generated, hence generate token from user password. In this case subLayer metadata update will fail + if(token == null || token.isBlank()) + { + token = generateToken(60); + } } - // check if item exists ItemEntry itemEntry = searchForItem(resourceUrl); if (itemEntry == null) { - // add item if doesn't exist - // add item with dummy 'simple' metadata - // no longer needed, but keep for testing purposes - /* - String dummyMetadata = "" - + "" - + " " - + " 2022-07-21" - + " 12:26:34.68" - + " 2022-07-21" - + " 12:59:19.59" - + " editor:esri.dijit.metadata.editor" - + " 1.0" - + " ISO 19139 Metadata Implementation Specification GML3.2" - + " ISO19139" - + " false" - + " " - + " " - + " " + resourceUrl + "" - + " 2016-02-1916:48:06.84" - + " " - + " " - + " " + title + "" - + " " - + " " + sanitize(description) + "" - + " " - + ""; - - Path dummyMetadataPath = Files.createTempFile(null, null); - Files.write(dummyMetadataPath, dummyMetadata.getBytes(StandardCharsets.UTF_8)); - File dummyMetadataFile = dummyMetadataPath.toFile(); - */ - - ItemResponse response = addItem( - title, - description, - new URL(resourceUrl), - sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, - itemType, - extractEnvelope(bbox), - typeKeywords, - null, - metadataFile, - token - ); - - // remove the dummy metadata file - // no longer needed, but keep for testing purposes - // dummyMetadataFile.delete(); - - if (response == null || !response.success) { - String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; - throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); - } else { - System.out.print("addItem -> " + response.toString()); - - // Now upload full metadata - String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); - System.out.print("METADATA -> " + metadataAdded); - } - - client.share(definition.getCredentials().getUserName(), definition.getFolderId(), response.id, true, true, null, token); + // neither the potential sub layer, nor the parent layer exist. Add a new portal item + + ItemResponse response = addItem( + title, + description, + new URL(resourceUrl), + sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, + itemType, + extractEnvelope(bbox), + typeKeywords, + null, + metadataFile, + token + ); + + if (response == null || !response.success) { + String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; + throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); + } else { + System.out.print("addItem -> " + response.toString()); + + // Now upload full metadata + String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); + System.out.print("METADATA -> " + metadataAdded); + } - return PublishingStatus.CREATED; + client.share(definition.getCredentials().getUserName(), definition.getFolderId(), response.id, true, true, null, token); - } else { // if (itemEntry.owner.equals(definition.getCredentials().getUserName())) { - // if the item is not owned by the registered account, try to update - // assuming the item is in a shared update group. + return PublishingStatus.CREATED; + + } else { + // there is an item registered for this resourceUrl, try to update + // if the item is not owned by the registered account + // assume the item is in a shared update group. // if this fails and the registered account cannot update the existing item - // then consider this item 'skipped' + // then this item will be 'skipped' itemEntry = client.readItem(itemEntry.id, token); if (itemEntry == null) { @@ -373,33 +353,53 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } // update item if does exist - ItemResponse response = updateItem( - itemEntry.id, - itemEntry.owner, - itemEntry.ownerFolder, - title, - description, - new URL(resourceUrl), - sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, - itemType, - extractEnvelope(bbox), - typeKeywords, - fileToUpload, - metadataFile, - token - ); - - if (response == null || !response.success) { - String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; - throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); - } else { - // String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); - System.out.print("updateItem -> " + response.toString()); - } + + // if the item url is the same as the resourceUrl, proceed. + // otherwise the metadata is for a sublayer, but the item is the parent + if (resourceUrl.equals(itemEntry.url)) { + ItemResponse response = updateItem( + itemEntry.id, + itemEntry.owner, + itemEntry.ownerFolder, + title, + description, + new URL(resourceUrl), + sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, + itemType, + extractEnvelope(bbox), + typeKeywords, + fileToUpload, + metadataFile, + token + ); + + if (response == null || !response.success) { + String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; + throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); + } else { + // String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); + System.out.print("updateItem -> " + response.toString()); + } - existing.remove(itemEntry.id); + existing.remove(itemEntry.id); - return PublishingStatus.UPDATED; + return PublishingStatus.UPDATED; + } else { + // the metadata is apparently for a sublayer + // DO SOMETHING ELSE + String parentUrl = resourceUrl.substring(0,resourceUrl.lastIndexOf("/")); + String featureServerToken = generateToken(60, parentUrl,token); + String metadataUpdateURI = resourceUrl + "/metadata/update/"; + boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); + LOG.debug("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); + System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); + + if (wasUpdated) { + return PublishingStatus.UPDATED; + } else { + return PublishingStatus.SKIPPED; + } + } } } catch (MalformedURLException ex) { return PublishingStatus.SKIPPED; @@ -416,6 +416,23 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } } } + + + private PublishingStatus updateSubLayerMetadata(String resourceUrl, String arcgisMetadata) throws URISyntaxException, IOException { + String parentUrl = resourceUrl.substring(0,resourceUrl.lastIndexOf("/")); + String featureServerToken = generateToken(60, parentUrl,token); + String metadataUpdateURI = resourceUrl + "/metadata/update/"; + boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); + System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); + + if (wasUpdated) { + return PublishingStatus.UPDATED; + } else { + return PublishingStatus.SKIPPED; + } + } + + private ItemType createItemType(String resourceUrl) { resourceUrl = StringUtils.trimToEmpty(resourceUrl); @@ -450,6 +467,27 @@ private ItemEntry searchForItem(String src_uri_s) throws URISyntaxException, IOE QueryResponse search = client.search(String.format("url:%s", String.format("%s", src_uri_s)), 0, 0, token); ItemEntry itemEntry = search != null && search.results != null && search.results.length > 0 ? search.results[0] : null; + // if no results found and the source uri (src_uri_s) ends in /0, /1, ... this may be a sublayer + // look for the parent service + if (itemEntry == null) { + String[] uri_parts = src_uri_s.split("/"); + + // iff the last part is a number, this could be a sublayer + if (StringUtils.isNumeric(uri_parts[uri_parts.length-1])) { + String regex = "\\/" + uri_parts[uri_parts.length-1] + "$"; + String mother_uri = ""; + try { + mother_uri = src_uri_s.replaceFirst(regex, ""); + } catch (Exception e) { + System.out.println(e.toString()); + } + + // now search for mother uri + search = client.search(String.format("url:%s", String.format("%s", mother_uri)), 0, 0, token); + itemEntry = search != null && search.results != null && search.results.length > 0 ? search.results[0] : null; + } + } + return itemEntry; } @@ -504,33 +542,9 @@ private MapAttribute extractMapAttributes(DataReference ref, byte [] content) th String sXml = ""; if (ref.getContentType().contains(MimeType.APPLICATION_XML) || ref.getContentType().contains(MimeType.TEXT_XML)) { sXml = new String(ref.getContent(MimeType.APPLICATION_XML, MimeType.TEXT_XML), "UTF-8"); - //DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - //factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - //factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - //factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - //factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - //factory.setXIncludeAware(false); - //factory.setExpandEntityReferences(false); - //factory.setNamespaceAware(true); - //DocumentBuilder builder = factory.newDocumentBuilder(); - //doc = builder.parse(new InputSource(new StringReader(sXml))); } else if (content!=null) { sXml = new String(content, "UTF-8"); } - /* - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - factory.setFeature("http://xml.org/sax/features/validation", false); - factory.setFeature("http://apache.org/xml/features/validation/schema", false); - factory.setXIncludeAware(false); - factory.setExpandEntityReferences(false); - factory.setNamespaceAware(true); - DocumentBuilder builder = factory.newDocumentBuilder(); - doc = builder.parse(new InputSource(new StringReader(sXml))); - */ doc = stringToDoc(sXml); if (doc!=null) { @@ -695,6 +709,10 @@ private DeleteResponse deleteItem(String id, String owner, String folderId, Stri private String generateToken(int minutes) throws URISyntaxException, IOException { return client.generateToken(minutes).token; } + private String generateToken(int minutes,String serverUrl,String token) throws URISyntaxException, IOException { + return client.generateToken(minutes,serverUrl,token).token; + } + private String generateToken() throws URISyntaxException, IOException { return client.generateToken(60).token; @@ -819,6 +837,24 @@ private FileName getFileNameFromUrl(String resourceUrl) { return new FileName(name, ext); } + private String replaceSpecialCharacters(String inputText) { + String outputText = ""; + String sourceText = inputText; + + HashMap extendedCharacters = new HashMap<>(); + + extendedCharacters.put("\\x91", "‘"); + extendedCharacters.put("\\x92", "’"); + extendedCharacters.put("\\x93", "“"); + extendedCharacters.put("\\x94", "”"); + + for (HashMap.Entry val: extendedCharacters.entrySet()) { + outputText = sourceText.replaceAll(val.getKey(), val.getValue()); + sourceText = outputText; + } + return outputText; + } + private static class FileName { public final String name; public final String ext; diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java index 680a70502..eb3ef6d3f 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java @@ -42,6 +42,8 @@ private Integer maxRedirects; private boolean uploadFiles; private boolean markdown2html; + private boolean useOAuth; + private String oAuthToken; /** @@ -71,6 +73,8 @@ public AgpOutputBrokerDefinitionAdaptor(EntityDefinition def) throws InvalidDefi maxRedirects = NumberUtils.toInt(get(P_MAX_REDIRECTS), DEFAULT_MAX_REDIRECTS); uploadFiles = Boolean.parseBoolean(get(P_UPLOAD)); markdown2html = Boolean.parseBoolean(get(P_MARKDOWN2HTML)); + useOAuth = Boolean.parseBoolean(get(P_OAUTH)); + oAuthToken = get(P_TOKEN); } } @@ -82,6 +86,7 @@ public void override(Map params) { consume(params, P_MAX_REDIRECTS); consume(params, P_UPLOAD); consume(params, P_MARKDOWN2HTML); + consume(params, P_TOKEN); credAdaptor.override(params); } @@ -194,4 +199,17 @@ public void setMarkdown2HTML(boolean markdown2html) { this.markdown2html = markdown2html; set(P_MARKDOWN2HTML, Boolean.toString(markdown2html)); } + + public boolean useOAuth() { + return useOAuth; + } + + public String getOAuthToken() + { + return oAuthToken; + } + public void setOAuthToken(String oAuthToken) { + this.oAuthToken = oAuthToken; + set(P_TOKEN, oAuthToken); + } } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java index e2e541fe1..a74c4592d 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java @@ -87,7 +87,7 @@ public String getHint() { return bundle.getString("agp.hint"); } }); - args.add(new UITemplate.StringArgument(P_FOLDER_ID, bundle.getString("agp.folderId"), false)); + args.add(new UITemplate.StringArgument(P_FOLDER_ID, bundle.getString("agp.folderId"), false)); args.add(new UITemplate.StringArgument(P_CRED_USERNAME, bundle.getString("agp.username"), true)); args.add(new UITemplate.StringArgument(P_CRED_PASSWORD, bundle.getString("agp.password"), true) { public boolean isPassword() { @@ -98,7 +98,8 @@ public boolean isPassword() { args.add(new UITemplate.BooleanArgument(P_FOLDER_CLEANUP, bundle.getString("agp.cleanup"))); args.add(new UITemplate.BooleanArgument(P_UPLOAD, bundle.getString("agp.upload"), true, true)); args.add(new UITemplate.BooleanArgument(P_MARKDOWN2HTML, bundle.getString("agp.markdown2html"), true, true)); - + args.add(new UITemplate.BooleanArgument(P_OAUTH, bundle.getString("agp.oauth"), false,false)); + args.add(new UITemplate.HiddenArgument(P_TOKEN, "hidden")); return new UITemplate(getType(), bundle.getString("agp"), args); } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties index c025d7902..53e9043b3 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties @@ -16,9 +16,10 @@ agp = Portal for ArcGIS agp.url = URL agp.folderId = Folder agp.username = User name -agp.password = user password +agp.password = User password agp.cleanup = Perform cleanup agp.max.redirects = Maximum redirects agp.hint = https://www.arcgis.com agp.upload = Upload files -agp.markdown2html = Markdown to HTML \ No newline at end of file +agp.markdown2html = Markdown to HTML +agp.oauth = Sign In to AGOL \ No newline at end of file diff --git a/geoportal-connectors/geoportal-harvester-agp-source/pom.xml b/geoportal-connectors/geoportal-harvester-agp-source/pom.xml index b359b12ff..21521e404 100644 --- a/geoportal-connectors/geoportal-harvester-agp-source/pom.xml +++ b/geoportal-connectors/geoportal-harvester-agp-source/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-agp-source Esri :: Geoportal Server :: Harvester :: Data Source :: ArcGIS Portal diff --git a/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java b/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java index 1880e8127..6fe7697ab 100644 --- a/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java @@ -196,7 +196,7 @@ public void terminate() { public DataContent readContent(String id) throws DataInputException { try { ItemEntry itemEntry = client.readItem(id, client.generateToken(1).token); - SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef()); + SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef(),itemEntry.title); ref.addContext(MimeType.APPLICATION_JSON, mapper.writeValueAsString(itemEntry).getBytes("UTF-8")); return ref; } catch (IOException|URISyntaxException ex) { @@ -228,7 +228,7 @@ private DataReference createReference(ItemEntry itemEntry) throws URISyntaxExcep props.put(WKAConstants.WKA_BBOX, sBox); } - SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef()); + SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef(),itemEntry.title); if (definition.getEmitXml()) { String orgMeta = null; diff --git a/geoportal-connectors/geoportal-harvester-ags/pom.xml b/geoportal-connectors/geoportal-harvester-ags/pom.xml index bffa03311..ab98819dc 100644 --- a/geoportal-connectors/geoportal-harvester-ags/pom.xml +++ b/geoportal-connectors/geoportal-harvester-ags/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-ags Esri :: Geoportal Server :: Harvester :: Data Source :: ArcGIS Server diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java index 4a630bb9f..c4acc7978 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java @@ -50,14 +50,22 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathFactory; import org.apache.commons.lang3.StringUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import com.esri.geoportal.commons.utils.XmlUtils; import com.esri.geoportal.geoportal.commons.geometry.GeometryService; import com.esri.geoportal.harvester.api.DataContent; @@ -69,10 +77,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.StringReader; import java.net.URL; import java.util.Arrays; import java.util.stream.Collectors; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; import org.apache.http.impl.client.LaxRedirectStrategy; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; /** * Ags broker. @@ -171,6 +184,7 @@ private ServerResponse layerInfoToServerResponse(LayerInfo layerInfo) { response.description = layerInfo.description; response.fullExtent = layerInfo.extent; response.initialExtent = layerInfo.extent; + response.metadataXML = layerInfo.metadataXML; return response; } @@ -253,6 +267,7 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx String itemInfoDescription = trimHtml(serverResponse.itemInfo!=null? serverResponse.itemInfo.description: null); String serverDescription = trimHtml(StringUtils.defaultString(StringUtils.defaultIfBlank(serverResponse.description, serverResponse.serviceDescription))); String description = StringUtils.defaultIfBlank(itemInfoDescription, serverDescription); + String metadataXML = serverResponse.metadataXML!=null? serverResponse.metadataXML: null; HashMap attributes = new HashMap<>(); attributes.put(WKAConstants.WKA_IDENTIFIER, new StringAttribute(serverResponse.url)); @@ -260,6 +275,7 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx attributes.put(WKAConstants.WKA_DESCRIPTION, new StringAttribute(description)); attributes.put(WKAConstants.WKA_RESOURCE_URL, new StringAttribute(serverResponse.url)); attributes.put(WKAConstants.WKA_RESOURCE_URL_SCHEME, new StringAttribute("urn:x-esri:specification:ServiceType:ArcGIS:" + (serviceType != null ? serviceType : "Unknown"))); + attributes.put(WKAConstants.WKA_METADATA_XML, new StringAttribute(metadataXML)); if (serverResponse.fullExtent != null) { normalizeExtent(serverResponse.fullExtent, 4326); @@ -269,10 +285,83 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx } } - MapAttribute attrs = new MapAttribute(attributes); - Document document = metaBuilder.create(attrs); - byte[] bytes = XmlUtils.toString(document).getBytes("UTF-8"); + Document document = null; + byte[] bytes; + + if ((this.definition.getUseServiceXml()) && (metadataXML != null && !metadataXML.trim().isEmpty())) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(false); + DocumentBuilder builder = null; + try { + builder = factory.newDocumentBuilder(); + document = builder.parse(new InputSource(new StringReader(metadataXML))); + + XPath xpath = XPathFactory.newInstance().newXPath(); + NodeList thumbnailNodes = (NodeList) xpath.compile("/metadata/Binary/Thumbnail/Data").evaluate(document, XPathConstants.NODESET); + if (thumbnailNodes.getLength() > 0) { + LOG.debug(thumbnailNodes.item(0).getTextContent()); + } + + NodeList serviceUrlNodes = (NodeList) xpath.compile("/metadata/distInfo/distributor/distorTran/onLineSrc/linkage").evaluate(document, XPathConstants.NODESET); + if (serviceUrlNodes.getLength() > 0) { + serviceUrlNodes.item(0).setNodeValue(serverResponse.url); + } else { + // no onLineSrc/linkage nodes + + // get /metadata + NodeList metadataNodes = (NodeList) xpath.compile("/metadata").evaluate(document, XPathConstants.NODESET); + + // get /metadata/distInfo + NodeList distinfoNodes = (NodeList) xpath.compile("/metadata/distinfo").evaluate(document, XPathConstants.NODESET); + if (distinfoNodes.getLength() == 0) { + Element distinfoElement = document.createElement("distinfo"); + metadataNodes.item(0).appendChild(distinfoElement); + } + + // get /metadata/distInfo/distributor + NodeList distributorNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor").evaluate(document, XPathConstants.NODESET); + if (distributorNodes.getLength() == 0) { + distinfoNodes = (NodeList) xpath.compile("/metadata/distinfo").evaluate(document, XPathConstants.NODESET); + Element distributorElement = document.createElement("distributor"); + distinfoNodes.item(0).appendChild(distributorElement); + } + + // get /metadata/distInfo/distributor/distorTran + NodeList distorTranNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran").evaluate(document, XPathConstants.NODESET); + if (distorTranNodes.getLength() == 0) { + distributorNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor").evaluate(document, XPathConstants.NODESET); + Element distorTranElement = document.createElement("distorTran"); + distributorNodes.item(0).appendChild(distorTranElement); + } + + // get /metadata/distInfo/distributor/distorTran/onLineSrc + NodeList onLineSrcNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran/onLineSrc").evaluate(document, XPathConstants.NODESET); + if (onLineSrcNodes.getLength() == 0) { + distorTranNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran").evaluate(document, XPathConstants.NODESET); + Element onLineSrcElement = document.createElement("onLineSrc"); + distorTranNodes.item(0).appendChild(onLineSrcElement); + } + + // get /metadata/distInfo/distributor/distorTran/onLineSrc/linkage + NodeList linkageNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran/onLineSrc/linkage").evaluate(document, XPathConstants.NODESET); + if (linkageNodes.getLength() == 0) { + onLineSrcNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran/onLineSrc").evaluate(document, XPathConstants.NODESET); + Element linkageElement = document.createElement("linkage"); + linkageElement.setTextContent(serverResponse.url); + onLineSrcNodes.item(0).appendChild(linkageElement); + } + } + + } catch (Exception ex) { + LOG.error(String.format("Error geting XML document. "), ex); + } + } else { + MapAttribute attrs = new MapAttribute(attributes); + document = metaBuilder.create(attrs); + } + bytes = XmlUtils.toString(document).getBytes("UTF-8"); + SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), getEntityDefinition().getLabel(), serverResponse.url, null, URI.create(serverResponse.url), td.getSource().getRef(), td.getRef()); attributes.entrySet().forEach(entry -> { ref.getAttributesMap().put(entry.getKey(), entry.getValue()); diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java index e8a2b99f1..ac87629bc 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java @@ -41,6 +41,7 @@ public class AgsBrokerDefinitionAdaptor extends BrokerDefinitionAdaptor { private boolean enableLayers; private boolean emitXml = true; private boolean emitJson = false; + private boolean useServiceXML = false; /** * Creates instance of the adaptor. @@ -64,6 +65,7 @@ public AgsBrokerDefinitionAdaptor(EntityDefinition def) throws InvalidDefinition enableLayers = BooleanUtils.toBoolean(get(P_ENABLE_LAYERS)); emitXml = BooleanUtils.toBooleanDefaultIfNull(BooleanUtils.toBooleanObject(get(P_EMIT_XML)), true); emitJson = BooleanUtils.toBooleanDefaultIfNull(BooleanUtils.toBooleanObject(get(P_EMIT_JSON)), false); + useServiceXML = BooleanUtils.toBooleanDefaultIfNull(BooleanUtils.toBooleanObject(get(P_USE_FULL_XML)), false); } } @@ -73,6 +75,7 @@ public void override(Map params) { consume(params,P_ENABLE_LAYERS); consume(params,P_EMIT_XML); consume(params,P_EMIT_JSON); + consume(params,P_USE_FULL_XML); credAdaptor.override(params); botsAdaptor.override(params); } @@ -161,4 +164,13 @@ public void setEmitJson(boolean emitJson) { set(P_EMIT_JSON, BooleanUtils.toStringTrueFalse(emitJson)); } + public boolean getUseServiceXml() { + return useServiceXML; + } + + public void setUseServiceXML(boolean useServiceXML) { + this.useServiceXML = useServiceXML; + set(P_USE_FULL_XML, BooleanUtils.toStringTrueFalse(useServiceXML)); + } + } diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java index 317bb400c..27c5769ac 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java @@ -71,6 +71,7 @@ public boolean isPassword() { args.add(new UITemplate.BooleanArgument(P_ENABLE_LAYERS, bundle.getString("ags.enableLayers"))); args.add(new UITemplate.BooleanArgument(P_EMIT_XML, bundle.getString("ags.emit.xml"),false, Boolean.TRUE)); args.add(new UITemplate.BooleanArgument(P_EMIT_JSON, bundle.getString("ags.emit.json"),false, Boolean.FALSE)); + args.add(new UITemplate.BooleanArgument(P_USE_FULL_XML, bundle.getString("ags.use_full_xml"),false, Boolean.FALSE)); return new UITemplate(getType(), bundle.getString("ags"), args); } diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java index ec03ad46d..1e93cd3f6 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java @@ -23,4 +23,5 @@ public static final String P_ENABLE_LAYERS = "ags-enable-layers"; public static final String P_EMIT_XML = "ags-emit-xml"; public static final String P_EMIT_JSON = "ags-emit-json"; + public static final String P_USE_FULL_XML = "ags-use-full-xml"; } diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties b/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties index f8bc2c729..1dd94a226 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties @@ -17,6 +17,7 @@ ags.url = URL ags.username = User name ags.password = User password ags.enableLayers = Enable layers -ags.hint = http://sampleserver1.arcgisonline.com/ArcGIS +ags.hint = https://services.arcgisonline.com/ArcGIS ags.emit.xml = Emit XML -ags.emit.json = Emit JSON \ No newline at end of file +ags.emit.json = Emit JSON +ags.use_full_xml = Use metadata instead of item info \ No newline at end of file diff --git a/geoportal-connectors/geoportal-harvester-ckan/pom.xml b/geoportal-connectors/geoportal-harvester-ckan/pom.xml index 82ca1671a..053003b26 100644 --- a/geoportal-connectors/geoportal-harvester-ckan/pom.xml +++ b/geoportal-connectors/geoportal-harvester-ckan/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-ckan jar diff --git a/geoportal-connectors/geoportal-harvester-console/pom.xml b/geoportal-connectors/geoportal-harvester-console/pom.xml index 20b546f3b..4e91cc789 100644 --- a/geoportal-connectors/geoportal-harvester-console/pom.xml +++ b/geoportal-connectors/geoportal-harvester-console/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-console Esri :: Geoportal Server :: Harvester :: Data Publisher :: Console diff --git a/geoportal-connectors/geoportal-harvester-csw/pom.xml b/geoportal-connectors/geoportal-harvester-csw/pom.xml index 73e083ce5..6c089cc2a 100644 --- a/geoportal-connectors/geoportal-harvester-csw/pom.xml +++ b/geoportal-connectors/geoportal-harvester-csw/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-csw Esri :: Geoportal Server :: Harvester :: Data Source :: Csw diff --git a/geoportal-connectors/geoportal-harvester-dcat/pom.xml b/geoportal-connectors/geoportal-harvester-dcat/pom.xml index de0619c56..c5104fe08 100644 --- a/geoportal-connectors/geoportal-harvester-dcat/pom.xml +++ b/geoportal-connectors/geoportal-harvester-dcat/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-dcat jar diff --git a/geoportal-connectors/geoportal-harvester-folder-big/pom.xml b/geoportal-connectors/geoportal-harvester-folder-big/pom.xml index dae5ce0ef..748e4272f 100644 --- a/geoportal-connectors/geoportal-harvester-folder-big/pom.xml +++ b/geoportal-connectors/geoportal-harvester-folder-big/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-folder-big Esri :: Geoportal Server :: Harvester :: Data Publisher :: Folder :: Big diff --git a/geoportal-connectors/geoportal-harvester-folder/pom.xml b/geoportal-connectors/geoportal-harvester-folder/pom.xml index a4d99059f..e4d247e0c 100644 --- a/geoportal-connectors/geoportal-harvester-folder/pom.xml +++ b/geoportal-connectors/geoportal-harvester-folder/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-folder Esri :: Geoportal Server :: Harvester :: Data Publisher :: Folder diff --git a/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java b/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java index 0837a7e8c..5f190c7ab 100644 --- a/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java +++ b/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java @@ -131,8 +131,9 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { try { for (MimeType ct: ref.getContentType()) { String extension = MimeTypeUtils.findExtensions(ct).stream().findFirst().orElse(null); - if (extension!=null) { - Path f = generateFileName(ref.getBrokerUri(), ref.getSourceUri(), ref.getId(), extension); + if (extension!=null) { + + Path f = generateFileName(ref.getBrokerUri(), ref.getSourceUri(), ref.getTitle() , extension); boolean created = !Files.exists(f); Files.createDirectories(f.getParent()); try (OutputStream output = Files.newOutputStream(f)) { @@ -178,7 +179,7 @@ private Path generateFileName(URI brokerUri, URI sourceUri, String id, String ex subFolder.remove(0); } for (String sf : subFolder) { - fileName = Paths.get(fileName.toString(), sf); + fileName = Paths.get(fileName.toString(), (id.isBlank() ? sf :id)); } if (!fileName.getFileName().toString().endsWith(extension)) { fileName = fileName.getParent().resolve(fileName.getFileName() + "." + extension); diff --git a/geoportal-connectors/geoportal-harvester-gpt/pom.xml b/geoportal-connectors/geoportal-harvester-gpt/pom.xml index 7d965aa4c..a269cb0d1 100644 --- a/geoportal-connectors/geoportal-harvester-gpt/pom.xml +++ b/geoportal-connectors/geoportal-harvester-gpt/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-gpt Esri :: Geoportal Server :: Harvester :: Data Publisher :: Geoportal Rest diff --git a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java index 0597553f4..484f70b46 100644 --- a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java +++ b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java @@ -47,7 +47,9 @@ import com.esri.geoportal.harvester.api.ex.DataProcessorException; import com.esri.geoportal.harvester.api.specs.OutputBroker; import com.esri.geoportal.harvester.api.specs.OutputConnector; +import java.util.ArrayList; import java.util.Arrays; +import java.util.ListIterator; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -122,10 +124,16 @@ public void onError(DataException ex) { public void terminate() { try { if (client != null && definition.getCleanup() && !preventCleanup) { + int deleted = 0; + for (String id : existing) { - client.delete(id); - } - LOG.info(String.format("%d records has been removed during cleanup.", existing.size())); + PublishResponse deleteResult = client.delete(id); + if ((deleteResult != null) && (deleteResult.getStatus() != null)) { + deleted += 1; + } + } + + LOG.info(String.format("%d records has been removed during cleanup.", deleted)); } } catch (URISyntaxException | IOException ex) { LOG.error(String.format("Error terminating broker."), ex); diff --git a/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml b/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml index 55547b73c..dabc93cff 100644 --- a/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml +++ b/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-gptsrc Esri :: Geoportal Server :: Harvester :: Data Source :: Geoportal Rest diff --git a/geoportal-connectors/geoportal-harvester-jdbc/pom.xml b/geoportal-connectors/geoportal-harvester-jdbc/pom.xml index dfda8aef3..c6c4383f2 100644 --- a/geoportal-connectors/geoportal-harvester-jdbc/pom.xml +++ b/geoportal-connectors/geoportal-harvester-jdbc/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-jdbc Esri :: Geoportal Server :: Harvester :: Data Source :: JDBC diff --git a/geoportal-connectors/geoportal-harvester-migration/pom.xml b/geoportal-connectors/geoportal-harvester-migration/pom.xml index c393e3733..ae1b2239d 100644 --- a/geoportal-connectors/geoportal-harvester-migration/pom.xml +++ b/geoportal-connectors/geoportal-harvester-migration/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-migration Esri :: Geoportal Server :: Harvester :: Data Source :: Migration Tool diff --git a/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml b/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml index b04d9e982..ef740bfc8 100644 --- a/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml +++ b/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-oai-pmh Esri :: Geoportal Server :: Harvester :: Data Source :: OAI-PMH diff --git a/geoportal-connectors/geoportal-harvester-sink/pom.xml b/geoportal-connectors/geoportal-harvester-sink/pom.xml index 1f35c05b1..39bc365a8 100644 --- a/geoportal-connectors/geoportal-harvester-sink/pom.xml +++ b/geoportal-connectors/geoportal-harvester-sink/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-sink Esri :: Geoportal Server :: Harvester :: Data Source :: Sink diff --git a/geoportal-connectors/geoportal-harvester-stac/pom.xml b/geoportal-connectors/geoportal-harvester-stac/pom.xml index 2b689e3fe..43043550a 100644 --- a/geoportal-connectors/geoportal-harvester-stac/pom.xml +++ b/geoportal-connectors/geoportal-harvester-stac/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-stac jar diff --git a/geoportal-connectors/geoportal-harvester-thredds/pom.xml b/geoportal-connectors/geoportal-harvester-thredds/pom.xml index 4f00e0385..bb80222df 100644 --- a/geoportal-connectors/geoportal-harvester-thredds/pom.xml +++ b/geoportal-connectors/geoportal-harvester-thredds/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-thredds jar diff --git a/geoportal-connectors/geoportal-harvester-unc/pom.xml b/geoportal-connectors/geoportal-harvester-unc/pom.xml index f8da7804b..6dab131eb 100644 --- a/geoportal-connectors/geoportal-harvester-unc/pom.xml +++ b/geoportal-connectors/geoportal-harvester-unc/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-unc Esri :: Geoportal Server :: Harvester :: Data Source :: Unc diff --git a/geoportal-connectors/geoportal-harvester-waf/pom.xml b/geoportal-connectors/geoportal-harvester-waf/pom.xml index 44a262462..a21eae8c3 100644 --- a/geoportal-connectors/geoportal-harvester-waf/pom.xml +++ b/geoportal-connectors/geoportal-harvester-waf/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-waf Esri :: Geoportal Server :: Harvester :: Data Source :: Waf diff --git a/geoportal-connectors/pom.xml b/geoportal-connectors/pom.xml index 526d4971a..1f07a6773 100644 --- a/geoportal-connectors/pom.xml +++ b/geoportal-connectors/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-connectors pom @@ -16,7 +16,6 @@ geoportal-harvester-dcat geoportal-harvester-folder geoportal-harvester-gpt - geoportal-harvester-waf geoportal-harvester-unc geoportal-harvester-ags geoportal-harvester-agp-publisher @@ -30,5 +29,6 @@ geoportal-harvester-jdbc geoportal-harvester-thredds geoportal-harvester-stac + geoportal-harvester-waf diff --git a/pom.xml b/pom.xml index dc4a12345..80be9e707 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.esri.geoportal geoportal-harvester - 2.7.0 + 2.7.1 pom Esri :: Geoportal Server :: Harvester Top-level project for all Harvester modules. @@ -47,7 +47,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.4.1 + 2.13.4.2 com.esri.geometry