From 1d63e49e708c9790798e68d863ef5a0516d5e736 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 13 Aug 2020 11:16:08 -0400 Subject: [PATCH 001/113] #6798 remove required=true to get BeanValidation messages --- src/main/webapp/dataverse.xhtml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 0c97b1a0ce2..1cd117df699 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -110,7 +110,8 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.name.title']}">
- + +
@@ -135,7 +136,7 @@
#{systemConfig.dataverseSiteUrl}/dataverse/
- +
@@ -164,7 +165,7 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.category.title']}">
- + @@ -191,7 +192,7 @@
- + From b12f635bece3e5f848b67d99f2f6b0cc5e12f499 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 17 Aug 2020 11:18:12 -0400 Subject: [PATCH 002/113] #6798 copy "required message" to Bundle --- src/main/java/propertyFiles/Bundle.properties | 4 ++++ src/main/webapp/dataverse.xhtml | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 8c70475953c..abb85d1dae7 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2115,6 +2115,10 @@ dataverse.link.error=Unable to link {0} to {1}. An internal error occurred. dataverse.search.user=Only authenticated users can save a search. dataverse.alias=alias dataverse.alias.taken=This Alias is already taken. +dataverse.name.required=Please enter a name. +dataverse.alias.required=Please enter an alias. +dataverse.category.required=Please select a category for your dataverse. + #editDatafilesPage.java dataset.save.fail=Dataset Save Failed diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 1cd117df699..b10cf1a2b57 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -95,7 +95,7 @@ @@ -110,7 +110,9 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.name.title']}">
- +
@@ -136,7 +138,10 @@
#{systemConfig.dataverseSiteUrl}/dataverse/
- +
@@ -165,7 +170,8 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.category.title']}">
- + @@ -386,7 +392,7 @@
- + @@ -495,7 +501,7 @@
  • - +
    • From 77d12da8789629e2b59c72e5159f79da2613427f Mon Sep 17 00:00:00 2001 From: ellenk Date: Wed, 19 Aug 2020 10:13:33 -0400 Subject: [PATCH 003/113] Updating existing API method for detecting orphan objects in the database, and in Solr. Added a new API method for clearing all orphans. (#4225) --- .../edu/harvard/iq/dataverse/api/Index.java | 61 ++++++++++---- .../iq/dataverse/search/IndexServiceBean.java | 84 ++++++++++++++----- .../search/SolrIndexServiceBean.java | 16 +--- 3 files changed, 113 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index c1f5f6957e6..6ab280a4042 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -384,15 +384,16 @@ public Response indexStatus() { JsonObjectBuilder contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr(); JsonObjectBuilder contentInSolrButNotDatabase; + JsonObjectBuilder permissionsInSolrButNotDatabase; try { contentInSolrButNotDatabase = getContentInSolrButNotDatabase(); - } catch (SearchException ex) { + permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase(); + } catch (SearchException ex) { return error(Response.Status.INTERNAL_SERVER_ERROR, "Can not determine index status. " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage()); } JsonObjectBuilder permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr(); - JsonObjectBuilder permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase(); - + JsonObjectBuilder data = Json.createObjectBuilder() .add("contentInDatabaseButStaleInOrMissingFromIndex", contentInDatabaseButStaleInOrMissingFromSolr) .add("contentInIndexButNotDatabase", contentInSolrButNotDatabase) @@ -401,7 +402,38 @@ public Response indexStatus() { return ok(data); } - + + @GET + @Path("clear-orphans") + public Response clearOrphans() { + try { + List dataversesInSolrOnly = indexService.findDataversesInSolrOnly(); + List datasetsInSolrOnly = indexService.findDatasetsInSolrOnly(); + List filesInSolrOnly = indexService.findFilesInSolrOnly(); + List permissionsInSolrOnly = indexService.findPermissionsInSolrOnly(); + + // Convert these Ids to solrIds + List solrIds = new ArrayList<>(); + for (Long id : dataversesInSolrOnly) { + solrIds.add(IndexServiceBean.solrDocIdentifierDataverse+id); + } + + for(Long id: datasetsInSolrOnly){ + solrIds.add(IndexServiceBean.solrDocIdentifierDataset+id); + } + for(Long id: filesInSolrOnly) { + solrIds.add(IndexServiceBean.solrDocIdentifierFile+id); + } + for(String id: permissionsInSolrOnly) { + solrIds.add(id); + } + IndexResponse resultOfSolrDeletionAttempt = solrIndexService.deleteMultipleSolrIds(solrIds); + return ok(resultOfSolrDeletionAttempt.getMessage()); + } catch(SearchException ex) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "Error deleting orphaned documents: " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage()); + } + + } private JsonObjectBuilder getContentInDatabaseButStaleInOrMissingFromSolr() { List stateOrMissingDataverses = indexService.findStaleOrMissingDataverses(); List staleOrMissingDatasets = indexService.findStaleOrMissingDatasets(); @@ -444,9 +476,9 @@ private JsonObjectBuilder getContentInSolrButNotDatabase() throws SearchExceptio * @todo What about files? Currently files are always indexed * along with their parent dataset */ - .add("dataverses", dataversesInSolrButNotDatabase.build().size()) - .add("datasets", datasetsInSolrButNotDatabase.build().size()) - .add("files", filesInSolrButNotDatabase.build().size()); + .add("dataverses", dataversesInSolrButNotDatabase.build()) + .add("datasets", datasetsInSolrButNotDatabase.build()) + .add("files", filesInSolrButNotDatabase.build()); return contentInSolrButNotDatabase; } @@ -458,17 +490,18 @@ private JsonObjectBuilder getPermissionsInDatabaseButStaleInOrMissingFromSolr() stalePermissionList.add(dvObjectId); } return Json.createObjectBuilder() - .add("dvobjects", stalePermissionList.build().size()); + .add("dvobjects", stalePermissionList.build()); } - - private JsonObjectBuilder getPermissionsInSolrButNotDatabase() { - List staleOrMissingPermissions = solrIndexService.findPermissionsInSolrNoLongerInDatabase(); + + private JsonObjectBuilder getPermissionsInSolrButNotDatabase() throws SearchException { + + List staleOrMissingPermissions = indexService.findPermissionsInSolrOnly(); JsonArrayBuilder stalePermissionList = Json.createArrayBuilder(); - for (Long dvObjectId : staleOrMissingPermissions) { - stalePermissionList.add(dvObjectId); + for (String id : staleOrMissingPermissions) { + stalePermissionList.add(id); } return Json.createObjectBuilder() - .add("dvobjects", stalePermissionList.build().size()); + .add("permissions", stalePermissionList.build()); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index b46c368e1d6..8a1a47a8b9b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -23,11 +23,9 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccessRequest; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.datavariable.DataVariable; -import edu.harvard.iq.dataverse.datavariable.VariableMetadata; import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; -import edu.harvard.iq.dataverse.harvest.server.OaiSetException; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; @@ -35,15 +33,12 @@ import java.io.IOException; import java.io.InputStream; import java.sql.Timestamp; -import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.Instant; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -60,13 +55,13 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import static javax.ejb.TransactionAttributeType.REQUIRES_NEW; -import javax.inject.Inject; import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; @@ -74,6 +69,7 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CursorMarkParams; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.io.IOUtils; import org.apache.tika.metadata.Metadata; @@ -1732,22 +1728,66 @@ public List findFilesInSolrOnly() throws SearchException { throw ex; } } - + /** + * Finds permissions documents in Solr that don't have corresponding dvObjects + * in the database, and returns a list of their Solr "id" field. + * @return list of "id" field vales for the orphaned Solr permission documents + * @throws SearchException + */ + public List findPermissionsInSolrOnly() throws SearchException { + List permissionInSolrOnly = new ArrayList<>(); + try { + int rows = 100; + SolrQuery q = (new SolrQuery(SearchFields.DEFINITION_POINT_DVOBJECT_ID+":*")).setRows(rows).setSort(SortClause.asc(SearchFields.ID)); + String cursorMark = CursorMarkParams.CURSOR_MARK_START; + boolean done = false; + while (!done) { + q.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); + QueryResponse rsp = solrServer.query(q); + String nextCursorMark = rsp.getNextCursorMark(); + SolrDocumentList list = rsp.getResults(); + for (SolrDocument doc: list) { + long id = Long.parseLong((String) doc.getFieldValue(SearchFields.DEFINITION_POINT_DVOBJECT_ID)); + DvObject dvobject = dvObjectService.findDvObject(id); + if (dvobject == null) { + permissionInSolrOnly.add((String)doc.getFieldValue(SearchFields.ID)); + } + } + if (cursorMark.equals(nextCursorMark)) { + done = true; + } + cursorMark = nextCursorMark; + } + } catch (SolrServerException | IOException ex) { + throw new SearchException("Error searching Solr for permissions" , ex); + + } + return permissionInSolrOnly; + } + private List findDvObjectInSolrOnly(String type) throws SearchException { SolrQuery solrQuery = new SolrQuery(); - solrQuery.setQuery("*"); - solrQuery.setRows(Integer.MAX_VALUE); + int rows = 100; + + solrQuery.setQuery("*").setRows(rows).setSort(SortClause.asc(SearchFields.ID)); solrQuery.addFilterQuery(SearchFields.TYPE + ":" + type); List dvObjectInSolrOnly = new ArrayList<>(); - QueryResponse queryResponse = null; - try { - queryResponse = solrClientService.getSolrClient().query(solrQuery); - } catch (SolrServerException | IOException ex) { - throw new SearchException("Error searching Solr for " + type, ex); - } - SolrDocumentList results = queryResponse.getResults(); - for (SolrDocument solrDocument : results) { - Object idObject = solrDocument.getFieldValue(SearchFields.ENTITY_ID); + + String cursorMark = CursorMarkParams.CURSOR_MARK_START; + boolean done = false; + while (!done) { + solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); + QueryResponse rsp = null; + try { + rsp = solrServer.query(solrQuery); + } catch (SolrServerException |IOException ex) { + throw new SearchException("Error searching Solr type: " + type, ex); + + } + String nextCursorMark = rsp.getNextCursorMark(); + SolrDocumentList list = rsp.getResults(); + for (SolrDocument doc: list) { + Object idObject = doc.getFieldValue(SearchFields.ENTITY_ID); if (idObject != null) { try { long id = (Long) idObject; @@ -1759,7 +1799,13 @@ private List findDvObjectInSolrOnly(String type) throws SearchException { throw new SearchException("Found " + SearchFields.ENTITY_ID + " but error casting " + idObject + " to long", ex); } } - } + } + if (cursorMark.equals(nextCursorMark)) { + done = true; + } + cursorMark = nextCursorMark; + } + return dvObjectInSolrOnly; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 14e9869aab3..f0974875067 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -22,16 +22,12 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; import javax.json.Json; import javax.json.JsonObjectBuilder; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; @@ -564,15 +560,5 @@ public List findPermissionsInDatabaseButStaleInOrMissingFromSolr() { return indexingRequired; } - /** - * @return A list of dvobject ids that should have their permissions - * re-indexed because Solr was down when a permission was revoked. The - * permission should be removed from Solr. - */ - public List findPermissionsInSolrNoLongerInDatabase() { - /** - * @todo Implement this! - */ - return new ArrayList<>(); - } + } From c4d8cf710731d7263c5f773a7aa77c4590edd505 Mon Sep 17 00:00:00 2001 From: ellenk Date: Thu, 20 Aug 2020 10:03:51 -0400 Subject: [PATCH 004/113] Add documentation for new API methods. (#4225) --- doc/sphinx-guides/source/api/native-api.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index e5fbdd79403..6a7bf1d5b5c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3059,6 +3059,20 @@ Note that if you are attempting to validate a very large number of datasets in y asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.request-timeout-seconds=3600 + + +Index and Database Consistency +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Get a list of all database objects that are missing in Solr, and Solr documents that are missing in the database:: + + curl $SERVER_URL/api/admin/index/status + +Remove all Solr documents that are orphaned (ie not associated with objects in the database):: + + curl $SERVER_URL/api/admin/index/clear-orphans + + Workflows ~~~~~~~~~ From 2caf3021d0cacd22e495df56acdef510ca420512 Mon Sep 17 00:00:00 2001 From: jingma Date: Fri, 21 Aug 2020 14:12:31 +0200 Subject: [PATCH 005/113] 5050 Parse all dc identifier elements and allow identifiers that don't have "doi" or "hdl" in them. --- .../api/imports/ImportGenericServiceBean.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 9b60993b365..9bbb01cf7aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -366,27 +366,38 @@ private FieldDTO makeDTO(DatasetFieldType dataverseFieldType, FieldDTO value, St } private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { + List otherIds = new ArrayList<>(); for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); MetadataBlockDTO value = entry.getValue(); if ("citation".equals(key)) { for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.otherId.equals(fieldDTO.getTypeName())) { - String otherId = ""; for (HashSet foo : fieldDTO.getMultipleCompound()) { for (FieldDTO next : foo) { if (DatasetFieldConstant.otherIdValue.equals(next.getTypeName())) { - otherId = next.getSinglePrimitive(); + otherIds.add(next.getSinglePrimitive()); } } - if (!otherId.isEmpty()){ - return otherId; - } } } } } } + if (!otherIds.isEmpty()) { + // We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F" + for (String otherId : otherIds) { + if (otherId.contains(GlobalId.DOI_PROTOCOL) || otherId.contains(GlobalId.HDL_PROTOCOL)) { + return otherId; + } + } + // But identifiers without hdl or doi like "10.6084/m9.figshare.12725075.v1" are also allowed + for (String otherId : otherIds) { + if (otherId.startsWith("10.") && otherId.contains("/")) { + return GlobalId.HDL_PROTOCOL + ":" + otherId; + } + } + } return null; } From 9cc6dedc34320608b86254c08f3f72f16e95ae46 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 26 Aug 2020 11:17:59 -0400 Subject: [PATCH 006/113] #7224 document purging the ingest queue with imqcmd --- .../source/admin/troubleshooting.rst | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index 0c752924b30..ec24de245b6 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -43,6 +43,26 @@ A User Needs Their Account to Be Converted From Institutional (Shibboleth), ORCI See :ref:`converting-shibboleth-users-to-local` and :ref:`converting-oauth-users-to-local`. +.. _troubleshooting-ingest: + +Ingest +------ + +Long-Running Ingest Jobs Have Exhausted System Resources +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ingest is both CPU- and memory-intensive, and depending on your system resources and the size and format of tabular data files uploaded, may render Dataverse unresponsive or nearly inoperable. It is possible to cancel these jobs by purging the ingest queue. + +``/usr/local/payara5/mq/bin/imqcmd -u admin query dst -t q -n DataverseIngest`` will query the DataverseIngest destination. The password, unless you have changed it, matches the username. + +``/usr/local/payara5/mq/bin/imqcmd -u admin purge dst -t q -n DataverseIngest`` will purge the DataverseIngest queue, and prompt for your confirmation. + +Finally, list destinations to verify that the purge was successful:: + +``/usr/local/payara5/mq/bin/imqcmd -u admin list dst`` + +If you are still running Glassfish, substitute glassfish4 for payara5 above. If you have installed Dataverse in some other location, adjust the above paths accordingly. + .. _troubleshooting-payara: Payara From 89cb145d90aba6509a11686d99fed6ef1b8bee83 Mon Sep 17 00:00:00 2001 From: jingma Date: Wed, 26 Aug 2020 23:14:47 +0200 Subject: [PATCH 007/113] Use handle library to resolve handle. --- .../dataverse/api/imports/ImportGenericServiceBean.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 9bbb01cf7aa..a91214b0dde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -40,6 +40,8 @@ import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.xml.stream.XMLInputFactory; +import net.handle.hdllib.HandleException; +import net.handle.hdllib.HandleResolver; /** @@ -393,8 +395,12 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { } // But identifiers without hdl or doi like "10.6084/m9.figshare.12725075.v1" are also allowed for (String otherId : otherIds) { - if (otherId.startsWith("10.") && otherId.contains("/")) { + try { + HandleResolver hr = new HandleResolver(); + hr.resolveHandle(otherId); return GlobalId.HDL_PROTOCOL + ":" + otherId; + } catch (HandleException e) { + logger.fine("Not a valid handle: " + e.toString()); } } } From 153b63efdf1ab581335e5b7da6a0a564c0b20116 Mon Sep 17 00:00:00 2001 From: jggautier Date: Thu, 27 Aug 2020 21:58:22 -0400 Subject: [PATCH 008/113] Update dataset-create-new-all-default-fields.json Closes #6671 Updates the example JSON file, dataset-create-new-all-default-fields.json, in the section of the API guides that describes how to create a dataset. The new JSON file includes two fields missing in the file in the latest API guides, topic classification and language. --- ...dataset-create-new-all-default-fields.json | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index ba801b9bae8..7a82cd4bb75 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -181,7 +181,7 @@ "typeName": "dsDescriptionValue", "multiple": false, "typeClass": "primitive", - "value": "DescriptionText 1" + "value": "DescriptionText1" }, "dsDescriptionDate": { "typeName": "dsDescriptionDate", @@ -264,6 +264,53 @@ } ] }, + { + "typeName": "topicClassification", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Term1" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Vocab1" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "https://TopicClassificationURL1.com" + } + }, + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Term2" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Vocab2" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "https://TopicClassificationURL2.com" + } + } + ] + }, { "typeName": "publication", "multiple": true, @@ -329,6 +376,15 @@ "typeClass": "primitive", "value": "Notes1" }, + { + "typeName": "language", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Abkhaz", + "Afar" + ] + }, { "typeName": "producer", "multiple": true, From c84ca49543fec04ca8e7958b6d4eb2f63aef071e Mon Sep 17 00:00:00 2001 From: jingma Date: Fri, 21 Aug 2020 14:12:31 +0200 Subject: [PATCH 009/113] 5050 Parse all dc identifier elements and allow identifiers that don't have "doi" or "hdl" in them. --- .../api/imports/ImportGenericServiceBean.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 9b60993b365..9bbb01cf7aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -366,27 +366,38 @@ private FieldDTO makeDTO(DatasetFieldType dataverseFieldType, FieldDTO value, St } private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { + List otherIds = new ArrayList<>(); for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); MetadataBlockDTO value = entry.getValue(); if ("citation".equals(key)) { for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.otherId.equals(fieldDTO.getTypeName())) { - String otherId = ""; for (HashSet foo : fieldDTO.getMultipleCompound()) { for (FieldDTO next : foo) { if (DatasetFieldConstant.otherIdValue.equals(next.getTypeName())) { - otherId = next.getSinglePrimitive(); + otherIds.add(next.getSinglePrimitive()); } } - if (!otherId.isEmpty()){ - return otherId; - } } } } } } + if (!otherIds.isEmpty()) { + // We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F" + for (String otherId : otherIds) { + if (otherId.contains(GlobalId.DOI_PROTOCOL) || otherId.contains(GlobalId.HDL_PROTOCOL)) { + return otherId; + } + } + // But identifiers without hdl or doi like "10.6084/m9.figshare.12725075.v1" are also allowed + for (String otherId : otherIds) { + if (otherId.startsWith("10.") && otherId.contains("/")) { + return GlobalId.HDL_PROTOCOL + ":" + otherId; + } + } + } return null; } From ab1375b5e7e943480f51abe4e53937786d0f546d Mon Sep 17 00:00:00 2001 From: jingma Date: Wed, 26 Aug 2020 23:14:47 +0200 Subject: [PATCH 010/113] Use handle library to resolve handle. --- .../dataverse/api/imports/ImportGenericServiceBean.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 9bbb01cf7aa..a91214b0dde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -40,6 +40,8 @@ import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.xml.stream.XMLInputFactory; +import net.handle.hdllib.HandleException; +import net.handle.hdllib.HandleResolver; /** @@ -393,8 +395,12 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { } // But identifiers without hdl or doi like "10.6084/m9.figshare.12725075.v1" are also allowed for (String otherId : otherIds) { - if (otherId.startsWith("10.") && otherId.contains("/")) { + try { + HandleResolver hr = new HandleResolver(); + hr.resolveHandle(otherId); return GlobalId.HDL_PROTOCOL + ":" + otherId; + } catch (HandleException e) { + logger.fine("Not a valid handle: " + e.toString()); } } } From ff50611d4756a0dcc7f02f18c2da8f0862825880 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 3 Sep 2020 12:01:25 -0400 Subject: [PATCH 011/113] don't rename files when replacing and draft version exists --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../dataverse/datasetutility/AddReplaceFileHelper.java | 2 +- .../harvard/iq/dataverse/ingest/IngestServiceBean.java | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 458fcf56ab0..ea2fac42cf2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3619,7 +3619,7 @@ public String save() { // have been created in the dataset. dataset = datasetService.find(dataset.getId()); - List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles); + List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, false); newFiles.clear(); // and another update command: diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 4928100dfff..ab34b5b2675 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -1501,7 +1501,7 @@ private boolean step_060_addFilesViaIngestService(){ } int nFiles = finalFileList.size(); - finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList); + finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, isFileReplaceOperation()); if (nFiles != finalFileList.size()) { if (nFiles == 1) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 6743c19875a..81edeb28077 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -153,7 +153,7 @@ public class IngestServiceBean { // DataFileCategory objects, if any were already assigned to the files). // It must be called before we attempt to permanently save the files in // the database by calling the Save command on the dataset and/or version. - public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles) { + public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, boolean isReplaceOperation) { List ret = new ArrayList<>(); if (newFiles != null && newFiles.size() > 0) { @@ -162,9 +162,10 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List Date: Thu, 3 Sep 2020 12:24:17 -0400 Subject: [PATCH 012/113] update method calls to match --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- .../iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 3138dcce2fe..be85a717070 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1128,7 +1128,7 @@ public String save() { } // Try to save the NEW files permanently: - List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles); + List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, false); // reset the working list of fileMetadatas, as to only include the ones // that have been added to the version successfully: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 6dfe605774f..23730885aab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -335,7 +335,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + violation.getMessage() + " The invalid value was \"" + violation.getInvalidValue() + "\"."); } else { - ingestService.saveAndAddFilesToDataset(editVersion, dataFiles); + ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, false); } } else { From e0b23ba6d249f4a2e2283f57abe53b470669cd06 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 4 Sep 2020 11:27:28 -0400 Subject: [PATCH 013/113] #6798 load val messages as a bundle for read msgs --- .../java/{ => propertyFiles}/ValidationMessages.properties | 0 .../{ => propertyFiles}/ValidationMessages_fr.properties | 0 src/main/webapp/dataverse.xhtml | 7 ++++--- 3 files changed, 4 insertions(+), 3 deletions(-) rename src/main/java/{ => propertyFiles}/ValidationMessages.properties (100%) rename src/main/java/{ => propertyFiles}/ValidationMessages_fr.properties (100%) diff --git a/src/main/java/ValidationMessages.properties b/src/main/java/propertyFiles/ValidationMessages.properties similarity index 100% rename from src/main/java/ValidationMessages.properties rename to src/main/java/propertyFiles/ValidationMessages.properties diff --git a/src/main/java/ValidationMessages_fr.properties b/src/main/java/propertyFiles/ValidationMessages_fr.properties similarity index 100% rename from src/main/java/ValidationMessages_fr.properties rename to src/main/java/propertyFiles/ValidationMessages_fr.properties diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index b10cf1a2b57..7a6fb8aca97 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -19,6 +19,7 @@ + @@ -112,7 +113,7 @@
    + requiredMessage="#{valMsg['dataverse.name']}"/>
    @@ -141,7 +142,7 @@ + required ="true" requiredMessage="#{valMsg['dataverse.alias']}"/>
    @@ -171,7 +172,7 @@
    + requiredMessage="#{valMsg['dataverse.category']}"> From 533aef3af7fb7b5e2318a375ffbf0523e25f2e87 Mon Sep 17 00:00:00 2001 From: ellenk Date: Tue, 8 Sep 2020 10:06:17 -0400 Subject: [PATCH 014/113] skip datafiles when checking for missing solr permissions (#4225) --- .../iq/dataverse/search/SolrIndexServiceBean.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index f0974875067..eaca51f559b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -528,12 +528,15 @@ public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServer } /** - * @todo Do we want to report the root dataverse (id 1, often) in - * permissionsInDatabaseButMissingFromSolr? + * * * @return A list of dvobject ids that should have their permissions * re-indexed Solr was down when a permission was added. The permission - * should be added to Solr. + * should be added to Solr. The id of the permission contains the type of + * DvObject and the primary key of the dvObject. + * DvObjects of type DataFile are currently skipped because their index + * time isn't stored in the database, since they are indexed along + * with their parent dataset (this may change). */ public List findPermissionsInDatabaseButStaleInOrMissingFromSolr() { List indexingRequired = new ArrayList<>(); @@ -543,7 +546,7 @@ public List findPermissionsInDatabaseButStaleInOrMissingFromSolr() { Timestamp permissionModificationTime = dvObject.getPermissionModificationTime(); Timestamp permissionIndexTime = dvObject.getPermissionIndexTime(); if (permissionIndexTime == null) { - if (dvObject.getId() != rootDvId) { + if (dvObject.getId() != rootDvId && !dvObject.isInstanceofDataFile()) { // we don't index the rootDv indexingRequired.add(dvObject.getId()); } From 4a42d2dde32d8d9b7f97e0022a3c357b91830c94 Mon Sep 17 00:00:00 2001 From: ellenk Date: Tue, 8 Sep 2020 10:07:10 -0400 Subject: [PATCH 015/113] report on number of missing objects, rather than ids, to be consistent with rest of the status message (#4225) --- src/main/java/edu/harvard/iq/dataverse/api/Index.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index 6ab280a4042..7ef9e399e62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -450,8 +450,8 @@ private JsonObjectBuilder getContentInDatabaseButStaleInOrMissingFromSolr() { * @todo What about files? Currently files are always indexed * along with their parent dataset */ - .add("dataverses", jsonStateOrMissingDataverses.build().size()) - .add("datasets", datasetsInDatabaseButNotSolr.build().size()); + .add("dataverses", jsonStateOrMissingDataverses.build()) + .add("datasets", datasetsInDatabaseButNotSolr.build()); return contentInDatabaseButStaleInOrMissingFromSolr; } From 125ea6fc1d83b14611eac3a7824412270d9ef1dc Mon Sep 17 00:00:00 2001 From: ellenk Date: Tue, 8 Sep 2020 10:07:46 -0400 Subject: [PATCH 016/113] add integration test for Index status api call (#4225) --- .../edu/harvard/iq/dataverse/api/IndexIT.java | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java diff --git a/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java b/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java new file mode 100644 index 00000000000..83c09c0ecf8 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java @@ -0,0 +1,122 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import static com.jayway.restassured.RestAssured.given; +import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.response.Response; +import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.logging.Logger; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.NO_CONTENT; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import static javax.ws.rs.core.Response.Status.OK; +import static junit.framework.Assert.assertEquals; +import org.hamcrest.CoreMatchers; +import static org.hamcrest.CoreMatchers.equalTo; +import org.junit.After; + +public class IndexIT { + + private static final Logger logger = Logger.getLogger(IndexIT.class.getCanonicalName()); + + @BeforeClass + public static void setUpClass() { + + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + Response makeSureTokenlessSearchIsEnabled = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiRequiresToken); + makeSureTokenlessSearchIsEnabled.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response remove = UtilIT.deleteSetting(SettingsServiceBean.Key.ThumbnailSizeLimitImage); + remove.then().assertThat() + .statusCode(200); + + } + + + @Test + public void testIndexStatus() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + Response getDatasetJsonNoFiles = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonNoFiles.prettyPrint(); + String protocol1 = JsonPath.from(getDatasetJsonNoFiles.getBody().asString()).getString("data.protocol"); + String authority1 = JsonPath.from(getDatasetJsonNoFiles.getBody().asString()).getString("data.authority"); + String identifier1 = JsonPath.from(getDatasetJsonNoFiles.getBody().asString()).getString("data.identifier"); + String dataset1PersistentId = protocol1 + ":" + authority1 + "/" + identifier1; + + Response uploadMd5File = UtilIT.uploadRandomFile(dataset1PersistentId, apiToken); + uploadMd5File.prettyPrint(); + assertEquals(CREATED.getStatusCode(), uploadMd5File.getStatusCode()); + + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/admin/index/status"); + response.prettyPrint(); + ArrayList emptyList = new ArrayList<>(); + HashMap data = response.getBody().jsonPath().get("data.contentInDatabaseButStaleInOrMissingFromIndex"); + System.out.println(data + " " + data.getClass()); + response.then().assertThat().statusCode(OK.getStatusCode()) + .body("data.contentInDatabaseButStaleInOrMissingFromIndex.dataverses", CoreMatchers.equalTo(emptyList)) + .body("data.contentInDatabaseButStaleInOrMissingFromIndex.datasets", CoreMatchers.equalTo(emptyList)) + .body("data.contentInIndexButNotDatabase.dataverses", CoreMatchers.equalTo(emptyList)) + .body("data.contentInIndexButNotDatabase.datasets", CoreMatchers.equalTo(emptyList)) + .body("data.contentInIndexButNotDatabase.files", CoreMatchers.equalTo(emptyList)) + .body("data.permissionsInDatabaseButStaleInOrMissingFromIndex.dvobjects", CoreMatchers.equalTo(emptyList)) + .body("data.permissionsInIndexButNotDatabase.permissions", CoreMatchers.equalTo(emptyList)); + + Response getDatasetJsonAfterMd5File = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonAfterMd5File.prettyPrint(); + getDatasetJsonAfterMd5File.then().assertThat() + .body("data.latestVersion.files[0].dataFile.md5", equalTo("0386269a5acb2c57b4eade587ff4db64")) + .body("data.latestVersion.files[0].dataFile.checksum.type", equalTo("MD5")) + .body("data.latestVersion.files[0].dataFile.checksum.value", equalTo("0386269a5acb2c57b4eade587ff4db64")); + + int fileId = JsonPath.from(getDatasetJsonAfterMd5File.getBody().asString()).getInt("data.latestVersion.files[0].dataFile.id"); + Response deleteFile = UtilIT.deleteFile(fileId, apiToken); + deleteFile.prettyPrint(); + deleteFile.then().assertThat() + .statusCode(NO_CONTENT.getStatusCode()); + + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + deleteDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + deleteDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + assertEquals(200, deleteUserResponse.getStatusCode()); + + } + + @After + public void tearDownDataverse() { + } + + @AfterClass + public static void cleanup() { + } + +} From fbdac70e25f59b83e54a76a2a6f55b7436af71d7 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 8 Sep 2020 10:49:46 -0400 Subject: [PATCH 017/113] adding release notes --- doc/release-notes/4225-stale-solr-records.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/4225-stale-solr-records.md diff --git a/doc/release-notes/4225-stale-solr-records.md b/doc/release-notes/4225-stale-solr-records.md new file mode 100644 index 00000000000..8f98b9843e7 --- /dev/null +++ b/doc/release-notes/4225-stale-solr-records.md @@ -0,0 +1,7 @@ +## Major Use Cases + +- Administrators will be able to use new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause search results to not load. + +## Notes for Dataverse Installation Administrators + +New API endpoints... point to Guides \ No newline at end of file From 2a5eae7ffd0dd81fded1ffed05b4cffc3a983217 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 9 Sep 2020 15:40:24 -0400 Subject: [PATCH 018/113] #6798 add required message for dv contact email --- .../ValidationMessages.properties | 1 + .../ValidationMessages_fr.properties | 49 ------------------- src/main/webapp/dataverse.xhtml | 3 +- 3 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 src/main/java/propertyFiles/ValidationMessages_fr.properties diff --git a/src/main/java/propertyFiles/ValidationMessages.properties b/src/main/java/propertyFiles/ValidationMessages.properties index 9c4f69252cf..86414dddcc1 100644 --- a/src/main/java/propertyFiles/ValidationMessages.properties +++ b/src/main/java/propertyFiles/ValidationMessages.properties @@ -18,6 +18,7 @@ dataverse.aliasLength=Alias must be at most 60 characters. dataverse.aliasNotnumber=Alias should not be a number dataverse.nameIllegalCharacters=Found an illegal character(s). Valid characters are a-Z, 0-9, '_', and '-'. dataverse.category=Please select a category for your dataverse. +dataverse.contact=Please enter a valid email address. contenttype.slash=Content-Type must contain a slash setspec.notNumber=Setspec should not be a number setspec.maxLength=Setspec must be at most 30 characters. diff --git a/src/main/java/propertyFiles/ValidationMessages_fr.properties b/src/main/java/propertyFiles/ValidationMessages_fr.properties deleted file mode 100644 index 40c43b00969..00000000000 --- a/src/main/java/propertyFiles/ValidationMessages_fr.properties +++ /dev/null @@ -1,49 +0,0 @@ -user.firstName=Veuillez entrer votre prénom. -user.lastName=Veuillez entrer votre nom de famille. -user.invalidEmail=Veuillez entrer une adresse courriel valide. -user.enterUsername=Veuillez entrer un nom d'utilisateur. -user.usernameLength=Le nom d'utilisateur doit comporter entre 2 et 60 caractères. -user.illegalCharacters=Caractère(s) non valide(s) utilisé(s). Les caractères valides sont a-Z, 0-9, '_', '-' et '.'. - -user.enterNickname=Veuillez entrer un pseudonyme. -user.nicknameLength=Le pseudonyme ne peut excéder 30 caractères. -user.nicknameNotnumber=Le pseudonyme ne devrait pas être un nombre - -dataset.templatename=Veuillez ajouter un nom pour le modèle d'ensemble de données. -dataset.nameLength=Le nom ne peut excéder 255 caractères. - -dataverse.name=Veuillez entrer un nom. -dataverse.alias=Veuillez entrer un alias. -dataverse.aliasLength=L'alias ne peut excéder 60 caractères. -dataverse.aliasNotnumber=L'alias ne devrait pas être un nombre. -dataverse.nameIllegalCharacters=Caractère(s) non valide(s) utilisé(s). Les caractères valides sont a-Z, 0-9, '_', '-'. -dataverse.category=Veuillez sélectionner une catégorie pour votre dataverse. -contenttype.slash=Le type de contenu doit contenir une barre oblique. -setspec.notNumber=Le nom (Setspec) ne devrait pas être un nombre. -setspec.maxLength=Le nom (Setspec) ne peut excéder 30 caractères. - -role.name=Un rôle doit avoir un nom. -desc.maxLength=Le description ne peut excéder 255 caractères. -alias.maxLength=L'alias ne peut excéder 26 caractères. -alias.illegalCharacters=L'alias ne peut être vide. Les caractères valides sont a-Z, 0-9, '_', '-'. - -custom.response=Veuillez entrer la réponse. -custom.questiontext=Veuillez entrer le texte de la question. -filename.illegalCharacters=Le nom du fichier ne peut contenir aucun des caractères suivants\u00A0: \ / : * ? " < > | ; # . -directoryname.illegalCharacters=Le nom du répertoire ne peut pas être suivi ni précédé d'un caractère séparateur de fichiers. -filename.blank=Veuillez spécifier un nom de fichier. - - -map.layername=Veuillez spécifier un nom de couche cartographique. -map.layerlink=Veuillez spécifier un lien de couche cartographique. -map.link=Veuillez spécifier un lien de carte intégrée. -map.imagelink=Veuillez spécifier un lien d'image de carte. -map.username=Veuillez spécifier un nom d'utilisateur WorldMap. - -oauth.username=Veuillez entrer votre nom d'utilisateur. - -password.retype=Le nouveau mot de passe est vide\u00A0: veuillez le retaper à nouveau. -password.current=Veuillez entrer votre mot de passe actuel. -password.validate=Page de réinitialisation du mot de passe par défaut. - -guestbook.name=Saisir un nom pour le registre de visiteurs. diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 7a6fb8aca97..2562128e2f4 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -199,7 +199,8 @@
    - + From b2c1afcda479d817fea8a34743a635cf0b589e3b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 9 Sep 2020 16:12:02 -0400 Subject: [PATCH 019/113] #6798 remove duplicate validation messages --- src/main/java/propertyFiles/Bundle.properties | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index abb85d1dae7..04c2fb10b76 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2115,9 +2115,6 @@ dataverse.link.error=Unable to link {0} to {1}. An internal error occurred. dataverse.search.user=Only authenticated users can save a search. dataverse.alias=alias dataverse.alias.taken=This Alias is already taken. -dataverse.name.required=Please enter a name. -dataverse.alias.required=Please enter an alias. -dataverse.category.required=Please select a category for your dataverse. #editDatafilesPage.java From 76c731f2ae750f87c4455ea597f26df2b999e407 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 10 Sep 2020 09:28:32 -0400 Subject: [PATCH 020/113] #6798 defer empty email test until save doc. return valmessages to original location --- .../java/{propertyFiles => }/ValidationMessages.properties | 2 ++ src/main/webapp/dataverse.xhtml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) rename src/main/java/{propertyFiles => }/ValidationMessages.properties (99%) diff --git a/src/main/java/propertyFiles/ValidationMessages.properties b/src/main/java/ValidationMessages.properties similarity index 99% rename from src/main/java/propertyFiles/ValidationMessages.properties rename to src/main/java/ValidationMessages.properties index 86414dddcc1..d0b1547d890 100644 --- a/src/main/java/propertyFiles/ValidationMessages.properties +++ b/src/main/java/ValidationMessages.properties @@ -48,3 +48,5 @@ password.current=Please enter your current password. password.validate=Password reset page default email message. guestbook.name=Enter a name for the guestbook + + diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 2562128e2f4..6d73b95b949 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -19,7 +19,7 @@ - + @@ -199,8 +199,8 @@
    - + From 1adbf71db532ef0d4e4108e90a229195cffb42b1 Mon Sep 17 00:00:00 2001 From: ellenk Date: Thu, 10 Sep 2020 11:45:58 -0400 Subject: [PATCH 021/113] changes based on code review for #4225 --- doc/release-notes/4225-stale-solr-records.md | 4 ++++ .../source/admin/solr-search-index.rst | 14 ++++++++++++++ doc/sphinx-guides/source/api/native-api.rst | 14 -------------- .../java/edu/harvard/iq/dataverse/api/Index.java | 2 +- .../iq/dataverse/search/SolrIndexServiceBean.java | 2 +- 5 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 doc/release-notes/4225-stale-solr-records.md diff --git a/doc/release-notes/4225-stale-solr-records.md b/doc/release-notes/4225-stale-solr-records.md new file mode 100644 index 00000000000..4b53651084c --- /dev/null +++ b/doc/release-notes/4225-stale-solr-records.md @@ -0,0 +1,4 @@ +## Notes for Dataverse Installation Administrators + +New API endpoints for reporting and clearing inconsistencies between Solr +and database - http://guides.dataverse.org/en/latest/admin/solr-search-index.html diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index 07e51b4564f..172db56c866 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -14,6 +14,20 @@ There are two ways to perform a full reindex of the Dataverse search index. Star Clear and Reindex +++++++++++++++++ + +Index and Database Consistency +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Get a list of all database objects that are missing in Solr, and Solr documents that are missing in the database:: + + curl $SERVER_URL/api/admin/index/status + +Remove all Solr documents that are orphaned (ie not associated with objects in the database):: + + curl $SERVER_URL/api/admin/index/clear-orphans + + + Clearing Data from Solr ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 1f62fa9e1fc..03e8f5f3f39 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3061,20 +3061,6 @@ Note that if you are attempting to validate a very large number of datasets in y asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.request-timeout-seconds=3600 - - -Index and Database Consistency -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Get a list of all database objects that are missing in Solr, and Solr documents that are missing in the database:: - - curl $SERVER_URL/api/admin/index/status - -Remove all Solr documents that are orphaned (ie not associated with objects in the database):: - - curl $SERVER_URL/api/admin/index/clear-orphans - - Workflows ~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index 7ef9e399e62..c4a489618b7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -388,7 +388,7 @@ public Response indexStatus() { try { contentInSolrButNotDatabase = getContentInSolrButNotDatabase(); permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase(); - } catch (SearchException ex) { + } catch (SearchException ex) { return error(Response.Status.INTERNAL_SERVER_ERROR, "Can not determine index status. " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index eaca51f559b..1eab2f86bba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -531,7 +531,7 @@ public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServer * * * @return A list of dvobject ids that should have their permissions - * re-indexed Solr was down when a permission was added. The permission + * re-indexed because Solr was down when a permission was added. The permission * should be added to Solr. The id of the permission contains the type of * DvObject and the primary key of the dvObject. * DvObjects of type DataFile are currently skipped because their index From 96e6be6af2c25a803e8b9d247dcfead5975969e0 Mon Sep 17 00:00:00 2001 From: ellenk Date: Thu, 10 Sep 2020 11:51:51 -0400 Subject: [PATCH 022/113] changes based on code review for #4225 - removed asserts for delete calls --- .../java/edu/harvard/iq/dataverse/api/IndexIT.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java b/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java index 83c09c0ecf8..0638acc524e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java @@ -97,18 +97,13 @@ public void testIndexStatus() { Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); - deleteDatasetResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); deleteDataverseResponse.prettyPrint(); - deleteDataverseResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - + Response deleteUserResponse = UtilIT.deleteUser(username); deleteUserResponse.prettyPrint(); - assertEquals(200, deleteUserResponse.getStatusCode()); - + } @After From 06905ec72b07f090a8299eb0c7f61f36e7a6387e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 10 Sep 2020 12:58:09 -0400 Subject: [PATCH 023/113] Adds folders to the file metadata passed to the outside "zipper" tool. #7255 --- .../edu/harvard/iq/dataverse/FileDownloadServiceBean.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index 683142fc5c4..a91a28d3d96 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.StringUtil; import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; @@ -563,6 +564,10 @@ public void addFileToCustomZipJob(String key, DataFile dataFile, Timestamp times fileName = dataFile.getFileMetadata().getLabel(); } } + + if (StringUtil.nonEmpty(dataFile.getFileMetadata().getDirectoryLabel())) { + fileName = dataFile.getFileMetadata().getDirectoryLabel() + "/" + fileName; + } if (location != null && fileName != null) { em.createNativeQuery("INSERT INTO CUSTOMZIPSERVICEREQUEST (KEY, STORAGELOCATION, FILENAME, ISSUETIME) VALUES (" From 9db0898165fc182fee62f3b81ba290dda39120d2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 10 Sep 2020 13:52:47 -0400 Subject: [PATCH 024/113] #6798 remove blank lines --- src/main/java/ValidationMessages.properties | 1 - src/main/java/propertyFiles/Bundle.properties | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ValidationMessages.properties b/src/main/java/ValidationMessages.properties index d0b1547d890..4dfce141f41 100644 --- a/src/main/java/ValidationMessages.properties +++ b/src/main/java/ValidationMessages.properties @@ -49,4 +49,3 @@ password.validate=Password reset page default email message. guestbook.name=Enter a name for the guestbook - diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 04c2fb10b76..8c70475953c 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2116,7 +2116,6 @@ dataverse.search.user=Only authenticated users can save a search. dataverse.alias=alias dataverse.alias.taken=This Alias is already taken. - #editDatafilesPage.java dataset.save.fail=Dataset Save Failed From ee30bddd9087235a5bf8f4178e395124a380dddd Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 14 Sep 2020 18:17:01 -0400 Subject: [PATCH 025/113] new get/set/delete APIs for file stores on datasets (#6872) --- .../harvard/iq/dataverse/api/Datasets.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 07d5944af05..c786a2da6dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -156,6 +156,7 @@ import org.glassfish.jersey.media.multipart.FormDataParam; import com.amazonaws.services.s3.model.PartETag; +import java.util.Map.Entry; @Path("datasets") public class Datasets extends AbstractApiBean { @@ -2204,6 +2205,57 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path return wr.getResponse(); } } + + @GET + @Path("{identifier}/filestore") + public Response getFileStore(@PathParam("identifier") String dvIdtf, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + Dataset dataset = findDatasetOrDie(dvIdtf); + + return response(req -> ok(dataset.getEffectiveStorageDriverId())); + } + + @PUT + @Path("{identifier}/filestore") + public Response setFileStore(@PathParam("identifier") String dvIdtf, + @PathParam("storeId") String fileStoreId, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + // Superuser-only: + AuthenticatedUser user = findAuthenticatedUserOrDie(); + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + + Dataset dataset = findDatasetOrDie(dvIdtf); + + // We don't want to allow setting this to a store id that does not exist: + for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) { + if(store.getKey().equals(fileStoreId)) { + dataset.setStorageDriverId(store.getValue()); + return ok("Storage set to: " + store.getKey() + "/" + store.getValue()); + } + } + return error(Response.Status.BAD_REQUEST, + "No Storage Driver found for : " + fileStoreId); + } + + @DELETE + @Path("{identifier}/filestore") + public Response resetFileStore(@PathParam("identifier") String dvIdtf, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + // Superuser-only: + AuthenticatedUser user = findAuthenticatedUserOrDie(); + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + + Dataset dataset = findDatasetOrDie(dvIdtf); + dataset.setStorageDriverId(null); + return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER); + } } From 5ef1710e914705e8f45dff9f6dd1fec807f21913 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 14 Sep 2020 18:36:24 -0400 Subject: [PATCH 026/113] adds the missing file types to the list of known saved original types. (#7265) --- .../harvard/iq/dataverse/util/FileUtil.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 02bf34f83c5..10aa8cced83 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -695,21 +695,27 @@ private static String checksumDigestToString(byte[] digestBytes) { } public static String generateOriginalExtension(String fileType) { - if (fileType.equalsIgnoreCase("application/x-spss-sav")) { return ".sav"; } else if (fileType.equalsIgnoreCase("application/x-spss-por")) { - return ".por"; - } else if (fileType.equalsIgnoreCase("application/x-stata")) { + return ".por"; + // in addition to "application/x-stata" we want to support + // "application/x-stata-13" ... etc.: + } else if (fileType.toLowerCase().startsWith("application/x-stata")) { return ".dta"; - } else if (fileType.equalsIgnoreCase( "application/x-rlang-transport")) { + } else if (fileType.equalsIgnoreCase("application/x-dvn-csvspss-zip")) { + return ".zip"; + } else if (fileType.equalsIgnoreCase("application/x-dvn-tabddi-zip")) { + return ".zip"; + } else if (fileType.equalsIgnoreCase("application/x-rlang-transport")) { return ".RData"; - } else if (fileType.equalsIgnoreCase("text/csv")) { + } else if (fileType.equalsIgnoreCase("text/csv") || fileType.equalsIgnoreCase("text/comma-separated-values")) { return ".csv"; - } else if (fileType.equalsIgnoreCase( "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) { + } else if (fileType.equalsIgnoreCase("text/tsv") || fileType.equalsIgnoreCase("text/tab-separated-values")) { + return ".tsv"; + } else if (fileType.equalsIgnoreCase("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) { return ".xlsx"; } - return ""; } From 246250d51fc12a2c6e21978a2b3badf765d0a9cb Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 15 Sep 2020 16:11:37 -0400 Subject: [PATCH 027/113] renamed the new apis, for consistency; per discussion earlier. #6872 --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index c786a2da6dc..67b3b9fe1d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2207,7 +2207,7 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path } @GET - @Path("{identifier}/filestore") + @Path("{identifier}/storageDriver") public Response getFileStore(@PathParam("identifier") String dvIdtf, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { @@ -2217,9 +2217,9 @@ public Response getFileStore(@PathParam("identifier") String dvIdtf, } @PUT - @Path("{identifier}/filestore") + @Path("{identifier}/storageDriver") public Response setFileStore(@PathParam("identifier") String dvIdtf, - @PathParam("storeId") String fileStoreId, + @PathParam("label") String storageDriverLabel, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { // Superuser-only: @@ -2232,17 +2232,17 @@ public Response setFileStore(@PathParam("identifier") String dvIdtf, // We don't want to allow setting this to a store id that does not exist: for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) { - if(store.getKey().equals(fileStoreId)) { + if(store.getKey().equals(storageDriverLabel)) { dataset.setStorageDriverId(store.getValue()); return ok("Storage set to: " + store.getKey() + "/" + store.getValue()); } } return error(Response.Status.BAD_REQUEST, - "No Storage Driver found for : " + fileStoreId); + "No Storage Driver found for : " + storageDriverLabel); } @DELETE - @Path("{identifier}/filestore") + @Path("{identifier}/storageDriver") public Response resetFileStore(@PathParam("identifier") String dvIdtf, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { From 3de07ec644ee2baddc4850f443315f5fe7cc9514 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 15 Sep 2020 17:27:18 -0400 Subject: [PATCH 028/113] code rearranged to have the dedicated storage driver for both dataverses and datasets (but not datafiles). #6872 --- .../edu/harvard/iq/dataverse/Dataverse.java | 30 +------------------ .../iq/dataverse/DvObjectContainer.java | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 75dbb39e2ca..b056101bb92 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -151,7 +151,7 @@ public String getIndexableCategoryName() { private String affiliation; - private String storageDriver=null; + ///private String storageDriver=null; // Note: We can't have "Remove" here, as there are role assignments that refer // to this role. So, adding it would mean violating a forign key contstraint. @@ -761,32 +761,4 @@ public boolean isAncestorOf( DvObject other ) { } return false; } - - public String getEffectiveStorageDriverId() { - String id = storageDriver; - if(StringUtils.isBlank(id)) { - if(this.getOwner() != null) { - id = this.getOwner().getEffectiveStorageDriverId(); - } else { - id= DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER; - } - } - return id; - } - - - public String getStorageDriverId() { - if(storageDriver==null) { - return DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER; - } - return storageDriver; - } - - public void setStorageDriverId(String storageDriver) { - if(storageDriver!=null&&storageDriver.equals(DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER)) { - this.storageDriver=null; - } else { - this.storageDriver = storageDriver; - } - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java index e40eb1c2a3a..f6b396f4c00 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java @@ -1,6 +1,8 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import javax.persistence.MappedSuperclass; +import org.apache.commons.lang.StringUtils; /** * A {@link DvObject} that can contain other {@link DvObject}s. @@ -26,4 +28,32 @@ public boolean isEffectivelyPermissionRoot() { return isPermissionRoot() || (getOwner() == null); } + private String storageDriver=null; + + public String getEffectiveStorageDriverId() { + String id = storageDriver; + if (StringUtils.isBlank(id)) { + if (this.getOwner() != null) { + id = this.getOwner().getEffectiveStorageDriverId(); + } else { + id = DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER; + } + } + return id; + } + + public String getStorageDriverId() { + if (storageDriver == null) { + return DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER; + } + return storageDriver; + } + + public void setStorageDriverId(String storageDriver) { + if (storageDriver != null && storageDriver.equals(DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER)) { + this.storageDriver = null; + } else { + this.storageDriver = storageDriver; + } + } } From 9ac5e8ce319afc0197c299958d673c8726c1edc1 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 15 Sep 2020 17:44:24 -0400 Subject: [PATCH 029/113] modifications needed to allow direct uploads when specifically enabled on a dataset (#6872) --- src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 2 +- .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 02bf34f83c5..58c1e193adc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1666,7 +1666,7 @@ public static boolean isPackageFile(DataFile dataFile) { } public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) { - String driverId = dataset.getDataverseContext().getEffectiveStorageDriverId(); + String driverId = dataset.getEffectiveStorageDriverId(); boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect"); //Should only be requested when it is allowed, but we'll log a warning otherwise if(!directEnabled) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 9c801f5197d..4699ec6c326 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DvObjectContainer; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinAuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider; @@ -1052,8 +1053,8 @@ public boolean isDatafileValidationOnPublishEnabled() { return settingsService.isTrueForKey(SettingsServiceBean.Key.FileValidationOnPublishEnabled, safeDefaultIfKeyNotFound); } - public boolean directUploadEnabled(Dataset dataset) { - return Boolean.getBoolean("dataverse.files." + dataset.getDataverseContext().getEffectiveStorageDriverId() + ".upload-redirect"); + public boolean directUploadEnabled(DvObjectContainer container) { + return Boolean.getBoolean("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"); } public String getDataCiteRestApiUrlString() { From 872ac52f0c51caa227e0cda3fe5e4e2523f21541 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 16 Sep 2020 09:12:53 -0400 Subject: [PATCH 030/113] other parts of the PR for #6872: flyway script, documentation. --- .../source/admin/dataverses-datasets.rst | 22 +++++++++++++++++++ doc/sphinx-guides/source/api/native-api.rst | 5 +++++ ...872-assign-storage-drivers-to-datasets.sql | 1 + 3 files changed, 28 insertions(+) create mode 100644 src/main/resources/db/migration/V5.0.0.1__6872-assign-storage-drivers-to-datasets.sql diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index 6349088beea..9c122c25abc 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -59,6 +59,8 @@ The available drivers can be listed with:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/storageDrivers +(Individual datasets can be configured to use specific file stores as well. See the "Datasets" section below.) + Datasets -------- @@ -130,3 +132,23 @@ Diagnose Constraint Violations Issues in Datasets To identify invalid data values in specific datasets (if, for example, an attempt to edit a dataset results in a ConstraintViolationException in the server log), or to check all the datasets in the Dataverse for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. +Configure a Dataset to store all new files in a specific file store +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configure a dataset to use a specific file store (this API can only be used by a superuser) :: + + curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/datasets/$dataset-id/storageDriver + +The current driver can be seen using:: + + curl http://$SERVER/api/datasets/$dataset-id/storageDriver + +It can be reset to the default store as follows (only a superuser can do this) :: + + curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$dataset-id/storageDriver + +The available drivers can be listed with:: + + curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/storageDrivers + + diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 03e8f5f3f39..bf778036f1b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1654,6 +1654,11 @@ The fully expanded example above (without environment variables) looks like this Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent dataverse in Solr. This endpoint requires the API token of a superuser. +Configure a Dataset to Use a Specific File Store +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/api/datasets/$dataset-id/storageDriver`` can be used to check, configure or reset the designated file store (storage driver) for a dataset. Please see the :doc:`/admin/dataverses-datasets` section of the guide for more information on this API. + Files ----- diff --git a/src/main/resources/db/migration/V5.0.0.1__6872-assign-storage-drivers-to-datasets.sql b/src/main/resources/db/migration/V5.0.0.1__6872-assign-storage-drivers-to-datasets.sql new file mode 100644 index 00000000000..453b2054c43 --- /dev/null +++ b/src/main/resources/db/migration/V5.0.0.1__6872-assign-storage-drivers-to-datasets.sql @@ -0,0 +1 @@ +ALTER TABLE dataset ADD COLUMN IF NOT EXISTS storagedriver VARCHAR(255); \ No newline at end of file From 66c00e8130fe81b79086f3a6de4ca8430a6b2483 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 16 Sep 2020 12:09:27 -0400 Subject: [PATCH 031/113] made the behavior of the APIs more consistent with the existing dataverse-level equivalents. (#6872) --- .../harvard/iq/dataverse/api/Datasets.java | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 67b3b9fe1d1..7f2cf982f55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2219,24 +2219,36 @@ public Response getFileStore(@PathParam("identifier") String dvIdtf, @PUT @Path("{identifier}/storageDriver") public Response setFileStore(@PathParam("identifier") String dvIdtf, - @PathParam("label") String storageDriverLabel, + String storageDriverLabel, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { // Superuser-only: - AuthenticatedUser user = findAuthenticatedUserOrDie(); + AuthenticatedUser user; + try { + user = findAuthenticatedUserOrDie(); + } catch (WrappedResponse ex) { + return error(Response.Status.BAD_REQUEST, "Authentication is required."); + } if (!user.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Superusers only."); } - Dataset dataset = findDatasetOrDie(dvIdtf); + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } // We don't want to allow setting this to a store id that does not exist: - for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) { - if(store.getKey().equals(storageDriverLabel)) { - dataset.setStorageDriverId(store.getValue()); - return ok("Storage set to: " + store.getKey() + "/" + store.getValue()); - } - } + for (Entry store : DataAccess.getStorageDriverLabels().entrySet()) { + if (store.getKey().equals(storageDriverLabel)) { + dataset.setStorageDriverId(store.getValue()); + datasetService.merge(dataset); + return ok("Storage driver set to: " + store.getKey() + "/" + store.getValue()); + } + } return error(Response.Status.BAD_REQUEST, "No Storage Driver found for : " + storageDriverLabel); } @@ -2247,14 +2259,26 @@ public Response resetFileStore(@PathParam("identifier") String dvIdtf, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { // Superuser-only: - AuthenticatedUser user = findAuthenticatedUserOrDie(); + AuthenticatedUser user; + try { + user = findAuthenticatedUserOrDie(); + } catch (WrappedResponse ex) { + return error(Response.Status.BAD_REQUEST, "Authentication is required."); + } if (!user.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Superusers only."); } - Dataset dataset = findDatasetOrDie(dvIdtf); - + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + dataset.setStorageDriverId(null); + datasetService.merge(dataset); return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER); } } From 8ae2fa9b23ea76c61ad988c4f60f2975986636d4 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 16 Sep 2020 12:49:48 -0400 Subject: [PATCH 032/113] adding release notes --- doc/release-notes/6872-dataset-specific-stores | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/release-notes/6872-dataset-specific-stores diff --git a/doc/release-notes/6872-dataset-specific-stores b/doc/release-notes/6872-dataset-specific-stores new file mode 100644 index 00000000000..f08aabe023f --- /dev/null +++ b/doc/release-notes/6872-dataset-specific-stores @@ -0,0 +1,8 @@ + +## Major Use Cases + +- Administrators will now be able to specify a store at the dataset level in addition to the Dataverse level (Issue #6872, PR #7272) + +## Notes for Administrators + +- New API available for setting a dataset store \ No newline at end of file From 50b12f343ba72b4a767cd22d8545f02f64c96a7e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 16 Sep 2020 14:30:34 -0400 Subject: [PATCH 033/113] Places in the code where the upload size limit and similar things are checked... all needed to be changed not to assume that it's inherited from the parent dataverse. (#6872) --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 4 ++-- .../edu/harvard/iq/dataverse/dataaccess/DataAccess.java | 7 ++++++- .../java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java | 2 +- .../engine/command/impl/AbstractCreateDatasetCommand.java | 2 +- src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 4 ++-- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index ea2fac42cf2..7f1a632a86a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1879,7 +1879,7 @@ private String init(boolean initFull) { //retrieveDatasetVersionResponse = datasetVersionService.retrieveDatasetVersionByVersionId(versionId); } - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); if (retrieveDatasetVersionResponse == null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index be85a717070..b4feecfcdf4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -444,7 +444,7 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo uploadedFiles = uploadedFilesList; selectedFiles = selectedFileMetadatasList; - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); logger.fine("done"); @@ -481,7 +481,7 @@ public String init() { - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); workingVersion = dataset.getEditVersion(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 0cf9883b240..0e2320401dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -20,6 +20,7 @@ package edu.harvard.iq.dataverse.dataaccess; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DvObject; import java.io.IOException; import java.util.HashMap; @@ -147,7 +148,11 @@ public static StorageIO createNewStorageIO(T dvObject, S throw new IOException("getDataAccessObject: null or invalid datafile."); } - return createNewStorageIO(dvObject, storageTag, dvObject.getDataverseContext().getEffectiveStorageDriverId()); + if (dvObject instanceof Dataset) { + return createNewStorageIO(dvObject, storageTag, ((Dataset)dvObject).getEffectiveStorageDriverId()); + } + // it's a DataFile: + return createNewStorageIO(dvObject, storageTag, dvObject.getOwner().getEffectiveStorageDriverId()); } public static StorageIO createNewStorageIO(T dvObject, String storageTag, String storageDriverId) throws IOException { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 9c61079e3f6..fb8f151f4c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -447,7 +447,7 @@ public static boolean isAppropriateStorageDriver(Dataset dataset){ // instead of testing for the 's3" store, //This method is used by both the dataset and edit files page so one change here //will fix both - return dataset.getDataverseContext().getEffectiveStorageDriverId().equals("s3"); + return dataset.getEffectiveStorageDriverId().equals("s3"); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java index 13652b93f75..2046e4b107e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java @@ -96,7 +96,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { theDataset.setAuthority(ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound)); } if (theDataset.getStorageIdentifier() == null) { - String driverId = theDataset.getDataverseContext().getEffectiveStorageDriverId(); + String driverId = theDataset.getEffectiveStorageDriverId(); theDataset.setStorageIdentifier(driverId + "://" + theDataset.getAuthorityForFileStorage() + "/" + theDataset.getIdentifierForFileStorage()); } if (theDataset.getIdentifier()==null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 58c1e193adc..0f0b0a7b1b1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -721,7 +721,7 @@ public static List createDataFiles(DatasetVersion version, InputStream // save the file, in the temporary location for now: Path tempFile = null; - Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getOwner().getEffectiveStorageDriverId()); + Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId()); String finalType = null; if (newStorageIdentifier == null) { if (getFilesTempDirectory() != null) { @@ -1331,7 +1331,7 @@ public static String getFilesTempDirectory() { } public static void generateS3PackageStorageIdentifier(DataFile dataFile) { - String driverId = dataFile.getDataverseContext().getEffectiveStorageDriverId(); + String driverId = dataFile.getOwner().getEffectiveStorageDriverId(); String bucketName = System.getProperty("dataverse.files." + driverId + ".bucket-name"); String storageId = driverId + "://" + bucketName + ":" + dataFile.getFileMetadata().getLabel(); From 49ebafc03f16961b29839c04719a7fcf7ef23a20 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 16 Sep 2020 16:28:40 -0400 Subject: [PATCH 034/113] better "no such dataset" error message in the GET version of the API. (#6872) --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 7f2cf982f55..655cdafe04c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2211,7 +2211,13 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path public Response getFileStore(@PathParam("identifier") String dvIdtf, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { - Dataset dataset = findDatasetOrDie(dvIdtf); + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } return response(req -> ok(dataset.getEffectiveStorageDriverId())); } From 3fc609255666dbd499610133691d11e2da25e9e8 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Thu, 17 Sep 2020 14:00:51 -0400 Subject: [PATCH 035/113] #7268 Catch abnormal EC2 exit and fail build --- tests/jenkins/ec2/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/jenkins/ec2/Jenkinsfile b/tests/jenkins/ec2/Jenkinsfile index 6d387c6e4db..4a16f865886 100644 --- a/tests/jenkins/ec2/Jenkinsfile +++ b/tests/jenkins/ec2/Jenkinsfile @@ -41,6 +41,13 @@ pipeline { sourcePattern: 'src/main/java', exclusionPattern: 'src/test*'] ) + script { + if (fileExists('./ansible_complete')) { + sh '/bin/rm ./ansible_complete' + } else { + error('Ansible run terminated abnormally, failing build.') + } + } } } } From 68adf561b3fc90197c25e5726ac89b945e198442 Mon Sep 17 00:00:00 2001 From: ellenk Date: Thu, 17 Sep 2020 16:45:58 -0400 Subject: [PATCH 036/113] Improved performance of API methods by making the SQL queries more efficient. Have the API methods return immediately, and report progress and method result in server.log. --- .../edu/harvard/iq/dataverse/Dataset.java | 4 + .../iq/dataverse/DatasetServiceBean.java | 11 +- .../edu/harvard/iq/dataverse/Dataverse.java | 2 + .../iq/dataverse/DataverseServiceBean.java | 7 + .../edu/harvard/iq/dataverse/DvObject.java | 2 + .../iq/dataverse/DvObjectServiceBean.java | 8 +- .../edu/harvard/iq/dataverse/api/Index.java | 130 ++-------------- .../search/IndexBatchServiceBean.java | 139 +++++++++++++++++- .../iq/dataverse/search/IndexServiceBean.java | 92 +++++------- .../search/SolrIndexServiceBean.java | 23 +-- 10 files changed, 219 insertions(+), 199 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 13a8692fdd4..4cf95dda250 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -39,6 +39,10 @@ * @author skraffmiller */ @NamedQueries({ + @NamedQuery(name = "Dataset.findIdStale", + query = "SELECT d.id FROM Dataset d WHERE d.indexTime is NULL OR d.indexTime < d.modificationTime"), + @NamedQuery(name = "Dataset.findIdStalePermission", + query = "SELECT d.id FROM Dataset d WHERE d.permissionIndexTime is NULL OR d.permissionIndexTime < d.permissionModificationTime"), @NamedQuery(name = "Dataset.findByIdentifier", query = "SELECT d FROM Dataset d WHERE d.identifier=:identifier"), @NamedQuery(name = "Dataset.findByIdentifierAuthorityProtocol", diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index c4049f3be00..c1efe119fd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -172,8 +172,15 @@ public List filterByPidQuery(String filterQuery) { public List findAll() { return em.createQuery("select object(o) from Dataset as o order by o.id", Dataset.class).getResultList(); } - - + + public List findIdStale() { + return em.createNamedQuery("Dataset.findIdStale").getResultList(); + } + + public List findIdStalePermission() { + return em.createNamedQuery("Dataset.findIdStalePermission").getResultList(); + } + public List findAllLocalDatasetIds() { return em.createQuery("SELECT o.id FROM Dataset o WHERE o.harvestedFrom IS null ORDER BY o.id", Long.class).getResultList(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 75dbb39e2ca..105c33d2d97 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -43,6 +43,8 @@ * @author mbarsinai */ @NamedQueries({ + @NamedQuery(name = "Dataverse.findIdStale",query = "SELECT d.id FROM Dataverse d WHERE d.indexTime is NULL OR d.indexTime < d.modificationTime"), + @NamedQuery(name = "Dataverse.findIdStalePermission",query = "SELECT d.id FROM Dataverse d WHERE d.permissionIndexTime is NULL OR d.permissionIndexTime < d.permissionModificationTime"), @NamedQuery(name = "Dataverse.ownedObjectsById", query = "SELECT COUNT(obj) FROM DvObject obj WHERE obj.owner.id=:id"), @NamedQuery(name = "Dataverse.findAll", query = "SELECT d FROM Dataverse d order by d.name"), @NamedQuery(name = "Dataverse.findRoot", query = "SELECT d FROM Dataverse d where d.owner.id=null"), diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 207dc829bf6..0e13936f0f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -127,6 +127,13 @@ public Dataverse find(Object pk) { public List findAll() { return em.createNamedQuery("Dataverse.findAll").getResultList(); } + + public List findIdStale() { + return em.createNamedQuery("Dataverse.findIdStale").getResultList(); + } + public List findIdStalePermission() { + return em.createNamedQuery("Dataverse.findIdStalePermission").getResultList(); + } /** * @param numPartitions The number of partitions you intend to split the diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java index 8a2f710c428..f1041303fdd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java @@ -20,6 +20,8 @@ query = "SELECT o FROM DvObject o ORDER BY o.id"), @NamedQuery(name = "DvObject.findById", query = "SELECT o FROM DvObject o WHERE o.id=:id"), + @NamedQuery(name = "DvObject.checkExists", + query = "SELECT count(o) from DvObject o WHERE o.id=:id"), @NamedQuery(name = "DvObject.ownedObjectsById", query="SELECT COUNT(obj) FROM DvObject obj WHERE obj.owner.id=:id"), @NamedQuery(name = "DvObject.findByGlobalId", diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java index 25f1d10f13f..4830c422d05 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java @@ -72,7 +72,13 @@ public List findByAuthenticatedUserId(AuthenticatedUser user) { query.setParameter("releaseUserId", user.getId()); return query.getResultList(); } - + + public boolean checkExists(Long id) { + Query query = em.createNamedQuery("DvObject.checkExists"); + query.setParameter("id", id); + Long result =(Long)query.getSingleResult(); + return result > 0; + } // FIXME This type-by-string has to go, in favor of passing a class parameter. public DvObject findByGlobalId(String globalIdString, String typeString) { return findByGlobalId(globalIdString, typeString, false); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index c4a489618b7..7aee225dc8e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -70,7 +70,7 @@ public class Index extends AbstractApiBean { @EJB IndexServiceBean indexService; @EJB - IndexBatchServiceBean indexAllService; + IndexBatchServiceBean indexBatchService; @EJB SolrIndexServiceBean solrIndexService; @EJB @@ -148,7 +148,7 @@ private Response indexAllOrSubset(Long numPartitionsSelected, Long partitionIdTo availablePartitionIdsBuilder.add(i); } - JsonObjectBuilder preview = indexAllService.indexAllOrSubsetPreview(numPartitions, partitionIdToProcess, skipIndexed); + JsonObjectBuilder preview = indexBatchService.indexAllOrSubsetPreview(numPartitions, partitionIdToProcess, skipIndexed); if (previewOnly) { preview.add("args", args); preview.add("availablePartitionIds", availablePartitionIdsBuilder); @@ -162,7 +162,7 @@ private Response indexAllOrSubset(Long numPartitionsSelected, Long partitionIdTo * @todo How can we expose the String returned from "index all" via * the API? */ - Future indexAllFuture = indexAllService.indexAllOrSubset(numPartitions, partitionIdToProcess, skipIndexed, previewOnly); + Future indexAllFuture = indexBatchService.indexAllOrSubset(numPartitions, partitionIdToProcess, skipIndexed, previewOnly); JsonObject workloadPreview = preview.build().getJsonObject("previewOfPartitionWorkload"); int dataverseCount = workloadPreview.getInt("dataverseCount"); int datasetCount = workloadPreview.getInt("datasetCount"); @@ -381,129 +381,17 @@ public Response indexPermissions(@PathParam("id") Long id) { @GET @Path("status") public Response indexStatus() { - JsonObjectBuilder contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr(); - - JsonObjectBuilder contentInSolrButNotDatabase; - JsonObjectBuilder permissionsInSolrButNotDatabase; - try { - contentInSolrButNotDatabase = getContentInSolrButNotDatabase(); - permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase(); - } catch (SearchException ex) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "Can not determine index status. " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage()); - } - - JsonObjectBuilder permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr(); - - JsonObjectBuilder data = Json.createObjectBuilder() - .add("contentInDatabaseButStaleInOrMissingFromIndex", contentInDatabaseButStaleInOrMissingFromSolr) - .add("contentInIndexButNotDatabase", contentInSolrButNotDatabase) - .add("permissionsInDatabaseButStaleInOrMissingFromIndex", permissionsInDatabaseButStaleInOrMissingFromSolr) - .add("permissionsInIndexButNotDatabase", permissionsInSolrButNotDatabase); - - return ok(data); + indexBatchService.indexStatus(); + return ok("Index Status Batch Job initiated, check log for job status."); } @GET @Path("clear-orphans") - public Response clearOrphans() { - try { - List dataversesInSolrOnly = indexService.findDataversesInSolrOnly(); - List datasetsInSolrOnly = indexService.findDatasetsInSolrOnly(); - List filesInSolrOnly = indexService.findFilesInSolrOnly(); - List permissionsInSolrOnly = indexService.findPermissionsInSolrOnly(); - - // Convert these Ids to solrIds - List solrIds = new ArrayList<>(); - for (Long id : dataversesInSolrOnly) { - solrIds.add(IndexServiceBean.solrDocIdentifierDataverse+id); - } - - for(Long id: datasetsInSolrOnly){ - solrIds.add(IndexServiceBean.solrDocIdentifierDataset+id); - } - for(Long id: filesInSolrOnly) { - solrIds.add(IndexServiceBean.solrDocIdentifierFile+id); - } - for(String id: permissionsInSolrOnly) { - solrIds.add(id); - } - IndexResponse resultOfSolrDeletionAttempt = solrIndexService.deleteMultipleSolrIds(solrIds); - return ok(resultOfSolrDeletionAttempt.getMessage()); - } catch(SearchException ex) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "Error deleting orphaned documents: " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage()); - } - - } - private JsonObjectBuilder getContentInDatabaseButStaleInOrMissingFromSolr() { - List stateOrMissingDataverses = indexService.findStaleOrMissingDataverses(); - List staleOrMissingDatasets = indexService.findStaleOrMissingDatasets(); - JsonArrayBuilder jsonStateOrMissingDataverses = Json.createArrayBuilder(); - for (Dataverse dataverse : stateOrMissingDataverses) { - jsonStateOrMissingDataverses.add(dataverse.getId()); - } - JsonArrayBuilder datasetsInDatabaseButNotSolr = Json.createArrayBuilder(); - for (Dataset dataset : staleOrMissingDatasets) { - datasetsInDatabaseButNotSolr.add(dataset.getId()); - } - JsonObjectBuilder contentInDatabaseButStaleInOrMissingFromSolr = Json.createObjectBuilder() - /** - * @todo What about files? Currently files are always indexed - * along with their parent dataset - */ - .add("dataverses", jsonStateOrMissingDataverses.build()) - .add("datasets", datasetsInDatabaseButNotSolr.build()); - return contentInDatabaseButStaleInOrMissingFromSolr; - } - - private JsonObjectBuilder getContentInSolrButNotDatabase() throws SearchException { - List dataversesInSolrOnly = indexService.findDataversesInSolrOnly(); - List datasetsInSolrOnly = indexService.findDatasetsInSolrOnly(); - List filesInSolrOnly = indexService.findFilesInSolrOnly(); - JsonArrayBuilder dataversesInSolrButNotDatabase = Json.createArrayBuilder(); - for (Long dataverseId : dataversesInSolrOnly) { - dataversesInSolrButNotDatabase.add(dataverseId); - } - JsonArrayBuilder datasetsInSolrButNotDatabase = Json.createArrayBuilder(); - for (Long datasetId : datasetsInSolrOnly) { - datasetsInSolrButNotDatabase.add(datasetId); - } - JsonArrayBuilder filesInSolrButNotDatabase = Json.createArrayBuilder(); - for (Long fileId : filesInSolrOnly) { - filesInSolrButNotDatabase.add(fileId); - } - JsonObjectBuilder contentInSolrButNotDatabase = Json.createObjectBuilder() - /** - * @todo What about files? Currently files are always indexed - * along with their parent dataset - */ - .add("dataverses", dataversesInSolrButNotDatabase.build()) - .add("datasets", datasetsInSolrButNotDatabase.build()) - .add("files", filesInSolrButNotDatabase.build()); - return contentInSolrButNotDatabase; + public Response clearOrphans() { + indexBatchService.clearOrphans(); + return ok("Clear Orphans Batch Job initiated, check log for job status."); } - - private JsonObjectBuilder getPermissionsInDatabaseButStaleInOrMissingFromSolr() { - List staleOrMissingPermissions; - staleOrMissingPermissions = solrIndexService.findPermissionsInDatabaseButStaleInOrMissingFromSolr(); - JsonArrayBuilder stalePermissionList = Json.createArrayBuilder(); - for (Long dvObjectId : staleOrMissingPermissions) { - stalePermissionList.add(dvObjectId); - } - return Json.createObjectBuilder() - .add("dvobjects", stalePermissionList.build()); - } - - private JsonObjectBuilder getPermissionsInSolrButNotDatabase() throws SearchException { - - List staleOrMissingPermissions = indexService.findPermissionsInSolrOnly(); - JsonArrayBuilder stalePermissionList = Json.createArrayBuilder(); - for (String id : staleOrMissingPermissions) { - stalePermissionList.add(id); - } - return Json.createObjectBuilder() - .add("permissions", stalePermissionList.build()); - } - + /** * We use the output of this method to generate our Solr schema.xml * diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java index 3746bb1d92c..fd9555eff89 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java @@ -1,12 +1,12 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.Future; import java.util.logging.Logger; @@ -44,11 +44,66 @@ public class IndexBatchServiceBean { DvObjectServiceBean dvObjectService; @EJB SystemConfig systemConfig; + + @Asynchronous + public Future indexStatus() { + JsonObjectBuilder response = Json.createObjectBuilder(); + logger.info("Beginning indexStatus()"); + JsonObjectBuilder contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr(); + JsonObjectBuilder contentInSolrButNotDatabase = null; + JsonObjectBuilder permissionsInSolrButNotDatabase = null; + try { + contentInSolrButNotDatabase = getContentInSolrButNotDatabase(); + permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase(); + + } catch (SearchException ex) { + String msg = "Can not determine index status. " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage(); + logger.info(msg); + response.add("SearchException ", msg); + return new AsyncResult<>(response); + } + + JsonObjectBuilder permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr(); + + JsonObjectBuilder data = Json.createObjectBuilder() + .add("contentInDatabaseButStaleInOrMissingFromIndex", contentInDatabaseButStaleInOrMissingFromSolr) + .add("contentInIndexButNotDatabase", contentInSolrButNotDatabase) + .add("permissionsInDatabaseButStaleInOrMissingFromIndex", permissionsInDatabaseButStaleInOrMissingFromSolr) + .add("permissionsInIndexButNotDatabase", permissionsInSolrButNotDatabase); + logger.info("indexStatus() result:" + data.build().toString()); + return new AsyncResult<>(data); + } + @Asynchronous + public Future clearOrphans() { + JsonObjectBuilder response = Json.createObjectBuilder(); + List solrIds = new ArrayList<>(); + logger.info("Beginning clearOrphans() to check for orphan Solr documents."); + try { + logger.info("checking for orphans type dataverse"); + solrIds.addAll(indexService.findDataversesInSolrOnly()); + logger.info("checking for orphans type dataset"); + solrIds.addAll(indexService.findDatasetsInSolrOnly()); + logger.info("checking for orphans file"); + solrIds.addAll(indexService.findFilesInSolrOnly()); + logger.info("checking for orphan permissions"); + solrIds.addAll(indexService.findPermissionsInSolrOnly()); + } catch (SearchException e) { + logger.info("SearchException in clearOrphans: " + e.getMessage()); + response.add("response from clearOrphans","SearchException: " + e.getMessage() ); + } + logger.info("found " + solrIds.size()+ " orphan documents"); + IndexResponse resultOfSolrDeletionAttempt = solrIndexService.deleteMultipleSolrIds(solrIds); + logger.info(resultOfSolrDeletionAttempt.getMessage()); + response.add("resultOfSolrDeletionAttempt", resultOfSolrDeletionAttempt.getMessage()); + + return new AsyncResult<>(response); + } + @Asynchronous public Future indexAllOrSubset(long numPartitions, long partitionId, boolean skipIndexed, boolean previewOnly) { JsonObjectBuilder response = Json.createObjectBuilder(); - Future responseFromIndexAllOrSubset = indexAllOrSubset(numPartitions, partitionId, skipIndexed); + indexAllOrSubset(numPartitions, partitionId, skipIndexed); String status = "indexAllOrSubset has begun"; response.add("responseFromIndexAllOrSubset", status); return new AsyncResult<>(response); @@ -221,5 +276,85 @@ public void indexDataverseRecursively(Dataverse dataverse) { } logger.info(dataverseIndexCount + " dataverses and " + datasetIndexCount + " datasets indexed. Total time to index " + (end - start) + "."); } + private JsonObjectBuilder getContentInDatabaseButStaleInOrMissingFromSolr() { + logger.info("checking for stale or missing dataverses"); + List stateOrMissingDataverses = indexService.findStaleOrMissingDataverses(); + logger.info("checking for stale or missing datasets"); + List staleOrMissingDatasets = indexService.findStaleOrMissingDatasets(); + JsonArrayBuilder jsonStaleOrMissingDataverses = Json.createArrayBuilder(); + for (Long id : stateOrMissingDataverses) { + jsonStaleOrMissingDataverses.add(id); + } + JsonArrayBuilder datasetsInDatabaseButNotSolr = Json.createArrayBuilder(); + for (Long id : staleOrMissingDatasets) { + datasetsInDatabaseButNotSolr.add(id); + } + JsonObjectBuilder contentInDatabaseButStaleInOrMissingFromSolr = Json.createObjectBuilder() + /** + * @todo What about files? Currently files are always indexed + * along with their parent dataset + */ + .add("dataverses", jsonStaleOrMissingDataverses.build()) + .add("datasets", datasetsInDatabaseButNotSolr.build()); + logger.info("completed check for stale or missing content."); + return contentInDatabaseButStaleInOrMissingFromSolr; + } + + private JsonObjectBuilder getContentInSolrButNotDatabase() throws SearchException { + logger.info("checking for dataverses in Solr only"); + List dataversesInSolrOnly = indexService.findDataversesInSolrOnly(); + logger.info("checking for datasets in Solr only"); + List datasetsInSolrOnly = indexService.findDatasetsInSolrOnly(); + logger.info("checking for files in Solr only"); + List filesInSolrOnly = indexService.findFilesInSolrOnly(); + JsonArrayBuilder dataversesInSolrButNotDatabase = Json.createArrayBuilder(); + logger.info("completed check for content in Solr but not database"); + for (String dataverseId : dataversesInSolrOnly) { + dataversesInSolrButNotDatabase.add(dataverseId); + } + JsonArrayBuilder datasetsInSolrButNotDatabase = Json.createArrayBuilder(); + for (String datasetId : datasetsInSolrOnly) { + datasetsInSolrButNotDatabase.add(datasetId); + } + JsonArrayBuilder filesInSolrButNotDatabase = Json.createArrayBuilder(); + for (String fileId : filesInSolrOnly) { + filesInSolrButNotDatabase.add(fileId); + } + JsonObjectBuilder contentInSolrButNotDatabase = Json.createObjectBuilder() + /** + * @todo What about files? Currently files are always indexed + * along with their parent dataset + */ + .add("dataverses", dataversesInSolrButNotDatabase.build()) + .add("datasets", datasetsInSolrButNotDatabase.build()) + .add("files", filesInSolrButNotDatabase.build()); + + return contentInSolrButNotDatabase; + } + + private JsonObjectBuilder getPermissionsInDatabaseButStaleInOrMissingFromSolr() { + List staleOrMissingPermissions; + logger.info("checking for permissions in database but stale or missing from Solr"); + staleOrMissingPermissions = solrIndexService.findPermissionsInDatabaseButStaleInOrMissingFromSolr(); + logger.info("completed checking for permissions in database but stale or missing from Solr"); + JsonArrayBuilder stalePermissionList = Json.createArrayBuilder(); + for (Long dvObjectId : staleOrMissingPermissions) { + stalePermissionList.add(dvObjectId); + } + return Json.createObjectBuilder() + .add("dvobjects", stalePermissionList.build()); + } + + private JsonObjectBuilder getPermissionsInSolrButNotDatabase() throws SearchException { + + List staleOrMissingPermissions = indexService.findPermissionsInSolrOnly(); + JsonArrayBuilder stalePermissionList = Json.createArrayBuilder(); + for (String id : staleOrMissingPermissions) { + stalePermissionList.add(id); + } + return Json.createObjectBuilder() + .add("permissions", stalePermissionList.build()); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 8a1a47a8b9b..2a166405a76 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1657,46 +1657,28 @@ private String getDesiredCardState(Map des /** * @return Dataverses that should be reindexed either because they have * never been indexed or their index time is before their modification time. + * (Exclude root because it is never indexed) */ - public List findStaleOrMissingDataverses() { - List staleDataverses = new ArrayList<>(); - for (Dataverse dataverse : dataverseService.findAll()) { - if (dataverse.equals(dataverseService.findRootDataverse())) { - continue; - } - if (stale(dataverse)) { - staleDataverses.add(dataverse); - } - } - return staleDataverses; + public List findStaleOrMissingDataverses() { + List staleDataverseIds = dataverseService.findIdStale(); + Long rootId = dataverseService.findRootDataverse().getId(); + List ids = new ArrayList<>(); + staleDataverseIds.stream().filter(id -> (!id.equals(rootId))).forEachOrdered(id -> { + ids.add(id); + }); + return ids; } /** * @return Datasets that should be reindexed either because they have never * been indexed or their index time is before their modification time. */ - public List findStaleOrMissingDatasets() { - List staleDatasets = new ArrayList<>(); - for (Dataset dataset : datasetService.findAll()) { - if (stale(dataset)) { - staleDatasets.add(dataset); - } - } - return staleDatasets; - } - - private boolean stale(DvObject dvObject) { - Timestamp indexTime = dvObject.getIndexTime(); - Timestamp modificationTime = dvObject.getModificationTime(); - if (indexTime == null) { - return true; - } else if (indexTime.before(modificationTime)) { - return true; - } - return false; + public List findStaleOrMissingDatasets() { + return datasetService.findIdStale(); } - public List findDataversesInSolrOnly() throws SearchException { + + public List findDataversesInSolrOnly() throws SearchException { try { /** * @todo define this centrally and statically @@ -1707,7 +1689,7 @@ public List findDataversesInSolrOnly() throws SearchException { } } - public List findDatasetsInSolrOnly() throws SearchException { + public List findDatasetsInSolrOnly() throws SearchException { try { /** * @todo define this centrally and statically @@ -1718,7 +1700,7 @@ public List findDatasetsInSolrOnly() throws SearchException { } } - public List findFilesInSolrOnly() throws SearchException { + public List findFilesInSolrOnly() throws SearchException { try { /** * @todo define this centrally and statically @@ -1748,8 +1730,7 @@ public List findPermissionsInSolrOnly() throws SearchException { SolrDocumentList list = rsp.getResults(); for (SolrDocument doc: list) { long id = Long.parseLong((String) doc.getFieldValue(SearchFields.DEFINITION_POINT_DVOBJECT_ID)); - DvObject dvobject = dvObjectService.findDvObject(id); - if (dvobject == null) { + if(!dvObjectService.checkExists(id)) { permissionInSolrOnly.add((String)doc.getFieldValue(SearchFields.ID)); } } @@ -1765,47 +1746,46 @@ public List findPermissionsInSolrOnly() throws SearchException { return permissionInSolrOnly; } - private List findDvObjectInSolrOnly(String type) throws SearchException { + private List findDvObjectInSolrOnly(String type) throws SearchException { SolrQuery solrQuery = new SolrQuery(); int rows = 100; solrQuery.setQuery("*").setRows(rows).setSort(SortClause.asc(SearchFields.ID)); solrQuery.addFilterQuery(SearchFields.TYPE + ":" + type); - List dvObjectInSolrOnly = new ArrayList<>(); - + List dvObjectInSolrOnly = new ArrayList<>(); + String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean done = false; while (!done) { solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = null; try { - rsp = solrServer.query(solrQuery); - } catch (SolrServerException |IOException ex) { + rsp = solrServer.query(solrQuery); + } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr type: " + type, ex); - + } String nextCursorMark = rsp.getNextCursorMark(); SolrDocumentList list = rsp.getResults(); for (SolrDocument doc: list) { - Object idObject = doc.getFieldValue(SearchFields.ENTITY_ID); - if (idObject != null) { - try { - long id = (Long) idObject; - DvObject dvobject = dvObjectService.findDvObject(id); - if (dvobject == null) { - dvObjectInSolrOnly.add(id); + Object idObject = doc.getFieldValue(SearchFields.ENTITY_ID); + if (idObject != null) { + try { + long id = (Long) idObject; + if (!dvObjectService.checkExists(id)) { + dvObjectInSolrOnly.add((String)doc.getFieldValue(SearchFields.ID)); + } + } catch (ClassCastException ex) { + throw new SearchException("Found " + SearchFields.ENTITY_ID + " but error casting " + idObject + " to long", ex); } - } catch (ClassCastException ex) { - throw new SearchException("Found " + SearchFields.ENTITY_ID + " but error casting " + idObject + " to long", ex); } } - } - if (cursorMark.equals(nextCursorMark)) { - done = true; - } - cursorMark = nextCursorMark; + if (cursorMark.equals(nextCursorMark)) { + done = true; } - + cursorMark = nextCursorMark; + } + return dvObjectInSolrOnly; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 1eab2f86bba..ef4422e8d89 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -541,25 +541,14 @@ public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServer public List findPermissionsInDatabaseButStaleInOrMissingFromSolr() { List indexingRequired = new ArrayList<>(); long rootDvId = dataverseService.findRootDataverse().getId(); - for (DvObject dvObject : dvObjectService.findAll()) { -// logger.info("examining dvObjectId " + dvObject.getId() + "..."); - Timestamp permissionModificationTime = dvObject.getPermissionModificationTime(); - Timestamp permissionIndexTime = dvObject.getPermissionIndexTime(); - if (permissionIndexTime == null) { - if (dvObject.getId() != rootDvId && !dvObject.isInstanceofDataFile()) { - // we don't index the rootDv - indexingRequired.add(dvObject.getId()); - } - } else if (permissionModificationTime == null) { - /** - * @todo What should we do here? Permissions should always be - * there. They are assigned at create time. - */ - logger.info("no permission modification time for dvobject id " + dvObject.getId()); - } else if (permissionIndexTime.before(permissionModificationTime)) { - indexingRequired.add(dvObject.getId()); + List missingDataversePermissionIds = dataverseService.findIdStalePermission(); + List missingDatasetPermissionIds = datasetService.findIdStalePermission(); + for (Long id : missingDataversePermissionIds) { + if (!id.equals(rootDvId)) { + indexingRequired.add(id); } } + indexingRequired.addAll(missingDatasetPermissionIds); return indexingRequired; } From 5c1400696ffc84f670723e504791e5ebaeb71d20 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Sun, 20 Sep 2020 18:18:07 -0400 Subject: [PATCH 037/113] Fixes the bug in the standalone zipper utility where it can't properly handle duplicate file names (IQSS/dataverse.harvard.edu#80) --- .../service/download/ZipDownloadService.java | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/scripts/zipdownload/src/main/java/edu/harvard/iq/dataverse/custom/service/download/ZipDownloadService.java b/scripts/zipdownload/src/main/java/edu/harvard/iq/dataverse/custom/service/download/ZipDownloadService.java index cda7cbb9505..4b66ee770d5 100644 --- a/scripts/zipdownload/src/main/java/edu/harvard/iq/dataverse/custom/service/download/ZipDownloadService.java +++ b/scripts/zipdownload/src/main/java/edu/harvard/iq/dataverse/custom/service/download/ZipDownloadService.java @@ -127,6 +127,7 @@ public void processFiles() { } Set zippedFolders = new HashSet<>(); + Set fileNamesList = new HashSet<>(); for (String [] fileEntry : jobFiles) { String storageLocation = fileEntry[0]; @@ -144,13 +145,15 @@ public void processFiles() { InputStream inputStream = this.directAccessUtil.openDirectAccess(storageLocation); - // (potential?) TODO: String zipEntryName = checkZipEntryName(fileName); + String zipEntryName = checkZipEntryName(fileName, fileNamesList); // this may not be needed anymore - some extra sanitizing of the file // name we used to have to do - since all the values in a current Dataverse - // database may already be santized enough. + // database may already be santized enough. + // (Edit: Yes, we still need this - there are still datasets with multiple + // files with duplicate names; this method takes care of that) if (inputStream != null && this.zipOutputStream != null) { - ZipEntry entry = new ZipEntry(fileName); + ZipEntry entry = new ZipEntry(zipEntryName); byte[] bytes = new byte[2 * 8192]; int read = 0; @@ -158,8 +161,8 @@ public void processFiles() { try { // Does this file have a folder name? - if (hasFolder(fileName)) { - addFolderToZipStream(getFolderName(fileName), zippedFolders); + if (hasFolder(zipEntryName)) { + addFolderToZipStream(getFolderName(zipEntryName), zippedFolders); } this.zipOutputStream.putNextEntry(entry); @@ -168,7 +171,6 @@ public void processFiles() { this.zipOutputStream.write(bytes, 0, read); readSize += read; } - inputStream.close(); this.zipOutputStream.closeEntry(); /*if (fileSize == readSize) { @@ -178,6 +180,12 @@ public void processFiles() { }*/ } catch (IOException ioex) { System.err.println("Failed to compress "+storageLocation); + } finally { + try { + inputStream.close(); + } catch (IOException ioexIgnore) { + System.err.println("Warning: IO exception trying to close input stream - "+storageLocation); + } } } else { System.err.println("Failed to access "+storageLocation); @@ -237,4 +245,21 @@ private void addFolderToZipStream(String folderName, Set zippedFolders) } } } + + // check for and process duplicates: + private String checkZipEntryName(String originalName, Set fileNames) { + String name = originalName; + int fileSuffix = 1; + int extensionIndex = originalName.lastIndexOf("."); + + while (fileNames.contains(name)) { + if (extensionIndex != -1) { + name = originalName.substring(0, extensionIndex) + "_" + fileSuffix++ + originalName.substring(extensionIndex); + } else { + name = originalName + "_" + fileSuffix++; + } + } + fileNames.add(name); + return name; + } } From 3fee1b8bd8d27880aa1e747780964173eb483e07 Mon Sep 17 00:00:00 2001 From: ellenk Date: Mon, 21 Sep 2020 11:11:28 -0400 Subject: [PATCH 038/113] make the indexStatus and clearOrphans optionally synchronous --- .../edu/harvard/iq/dataverse/api/Index.java | 51 +++++++++++++++++-- .../search/IndexBatchServiceBean.java | 20 +++++--- .../iq/dataverse/search/IndexServiceBean.java | 5 +- .../edu/harvard/iq/dataverse/api/IndexIT.java | 4 +- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index 7aee225dc8e..229cdebc4ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -43,7 +43,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Logger; import javax.ejb.EJB; import javax.ejb.EJBException; @@ -377,19 +380,57 @@ public Response indexPermissions(@PathParam("id") Long id) { return ok(indexResponse.getMessage()); } } - + /** + * Checks whether there are inconsistencies between the Solr index and + * the database, and reports back the status by content type + * @param sync - optional parameter, if set, then run the command + * synchronously. Else, return immediately, and report the status in server.log + * @return status report + */ @GET @Path("status") - public Response indexStatus() { - indexBatchService.indexStatus(); - return ok("Index Status Batch Job initiated, check log for job status."); + public Response indexStatus(@QueryParam("sync") String sync) { + Future result = indexBatchService.indexStatus(); + if (sync != null) { + try { + JsonObjectBuilder status = result.get(); + return ok(status); + } catch (InterruptedException | ExecutionException e) { + return AbstractApiBean.error(Status.INTERNAL_SERVER_ERROR, "indexStatus method interrupted: " + e.getLocalizedMessage()); + } + } else { + return ok("Index Status Batch Job initiated, check log for job status."); + } } - + /** + * Deletes "orphan" Solr documents (that don't match anything in the database). + * @param sync - optional parameter, if set, then run the command + * synchronously. Else, return immediately, and report the results in server.log + * @return what documents, if anything, was deleted + */ @GET @Path("clear-orphans") + /** + * Checks whether there are inconsistencies between the Solr index and + * the database, and reports back the status by content type + * @param sync - optional parameter, if !=null, then run the command + * synchronously. Else, return immediately, and report the status in server.log + * @return + */ public Response clearOrphans() { indexBatchService.clearOrphans(); return ok("Clear Orphans Batch Job initiated, check log for job status."); + Future result = indexBatchService.indexStatus(); + if (sync != null) { + try { + JsonObjectBuilder status = result.get(); + return ok(status); + } catch (InterruptedException | ExecutionException e) { + return AbstractApiBean.error(Status.INTERNAL_SERVER_ERROR, "indexStatus method interrupted: " + e.getLocalizedMessage()); + } + } else { + return ok("Index Status Batch Job initiated, check log for job status."); + } } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java index fd9555eff89..5171b1a864a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Future; +import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.AsyncResult; import javax.ejb.Asynchronous; @@ -49,12 +50,12 @@ public class IndexBatchServiceBean { public Future indexStatus() { JsonObjectBuilder response = Json.createObjectBuilder(); logger.info("Beginning indexStatus()"); - JsonObjectBuilder contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr(); - JsonObjectBuilder contentInSolrButNotDatabase = null; - JsonObjectBuilder permissionsInSolrButNotDatabase = null; + JsonObject contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr().build(); + JsonObject contentInSolrButNotDatabase = null; + JsonObject permissionsInSolrButNotDatabase = null; try { - contentInSolrButNotDatabase = getContentInSolrButNotDatabase(); - permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase(); + contentInSolrButNotDatabase = getContentInSolrButNotDatabase().build(); + permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase().build(); } catch (SearchException ex) { String msg = "Can not determine index status. " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage(); @@ -63,14 +64,19 @@ public Future indexStatus() { return new AsyncResult<>(response); } - JsonObjectBuilder permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr(); + JsonObject permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr().build(); JsonObjectBuilder data = Json.createObjectBuilder() .add("contentInDatabaseButStaleInOrMissingFromIndex", contentInDatabaseButStaleInOrMissingFromSolr) .add("contentInIndexButNotDatabase", contentInSolrButNotDatabase) .add("permissionsInDatabaseButStaleInOrMissingFromIndex", permissionsInDatabaseButStaleInOrMissingFromSolr) .add("permissionsInIndexButNotDatabase", permissionsInSolrButNotDatabase); - logger.info("indexStatus() result:" + data.build().toString()); + + logger.log(Level.INFO, "contentInDatabaseButStaleInOrMissingFromIndex: {0}", contentInDatabaseButStaleInOrMissingFromSolr); + logger.log(Level.INFO, "contentInIndexButNotDatabase: {0}", contentInSolrButNotDatabase); + logger.log(Level.INFO, "permissionsInDatabaseButStaleInOrMissingFromIndex: {0}", permissionsInDatabaseButStaleInOrMissingFromSolr); + logger.log(Level.INFO, "permissionsInIndexButNotDatabase: {0}", permissionsInSolrButNotDatabase); + return new AsyncResult<>(data); } @Asynchronous diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 2a166405a76..5b2d63c43eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -115,6 +115,9 @@ public class IndexServiceBean { @EJB VariableServiceBean variableService; + + @EJB + IndexBatchServiceBean indexBatchService; public static final String solrDocIdentifierDataverse = "dataverse_"; public static final String solrDocIdentifierFile = "datafile_"; @@ -157,7 +160,7 @@ public void close() { solrServer = null; } } - + @TransactionAttribute(REQUIRES_NEW) public Future indexDataverseInNewTransaction(Dataverse dataverse) throws SolrServerException, IOException{ return indexDataverse(dataverse, false); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java b/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java index 0638acc524e..40d1488ffc0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/IndexIT.java @@ -7,7 +7,6 @@ import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.util.ArrayList; -import java.util.HashMap; import java.util.logging.Logger; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.NO_CONTENT; @@ -68,11 +67,10 @@ public void testIndexStatus() { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) + .queryParam("sync","true") .get("/api/admin/index/status"); response.prettyPrint(); ArrayList emptyList = new ArrayList<>(); - HashMap data = response.getBody().jsonPath().get("data.contentInDatabaseButStaleInOrMissingFromIndex"); - System.out.println(data + " " + data.getClass()); response.then().assertThat().statusCode(OK.getStatusCode()) .body("data.contentInDatabaseButStaleInOrMissingFromIndex.dataverses", CoreMatchers.equalTo(emptyList)) .body("data.contentInDatabaseButStaleInOrMissingFromIndex.datasets", CoreMatchers.equalTo(emptyList)) From 5f1cc68f8bc0272d3a2ec53b363267eb506a769b Mon Sep 17 00:00:00 2001 From: ellenk Date: Mon, 21 Sep 2020 12:19:39 -0400 Subject: [PATCH 039/113] fix compile error in compareOrphans() --- .../edu/harvard/iq/dataverse/api/Index.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index 229cdebc4ff..406d4b98663 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -410,17 +410,17 @@ public Response indexStatus(@QueryParam("sync") String sync) { */ @GET @Path("clear-orphans") - /** - * Checks whether there are inconsistencies between the Solr index and - * the database, and reports back the status by content type - * @param sync - optional parameter, if !=null, then run the command - * synchronously. Else, return immediately, and report the status in server.log - * @return + /** + * Checks whether there are inconsistencies between the Solr index and the + * database, and reports back the status by content type + * + * @param sync - optional parameter, if !=null, then run the command + * synchronously. Else, return immediately, and report the status in + * server.log + * @return */ - public Response clearOrphans() { - indexBatchService.clearOrphans(); - return ok("Clear Orphans Batch Job initiated, check log for job status."); - Future result = indexBatchService.indexStatus(); + public Response clearOrphans(@QueryParam("sync") String sync) { + Future result = indexBatchService.clearOrphans(); if (sync != null) { try { JsonObjectBuilder status = result.get(); @@ -429,8 +429,8 @@ public Response clearOrphans() { return AbstractApiBean.error(Status.INTERNAL_SERVER_ERROR, "indexStatus method interrupted: " + e.getLocalizedMessage()); } } else { - return ok("Index Status Batch Job initiated, check log for job status."); - } + return ok("Clear Orphans Batch Job initiated, check log for job status."); + } } /** From ac7102638fbac0d9309b01a2f050fa98f1d0e64c Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 21 Sep 2020 15:32:06 -0400 Subject: [PATCH 040/113] Update Life Science metadata block. --- 6359-release-notes.md | 1 + conf/solr/7.7.2/schema_dv_mdb_copies.xml | 6 +- conf/solr/7.7.2/schema_dv_mdb_fields.xml | 6 +- .../api/data/metadatablocks/biomedical.tsv | 592 +++++++++--------- .../java/propertyFiles/biomedical.properties | 15 +- 5 files changed, 323 insertions(+), 297 deletions(-) create mode 100644 6359-release-notes.md diff --git a/6359-release-notes.md b/6359-release-notes.md new file mode 100644 index 00000000000..6901b17ce60 --- /dev/null +++ b/6359-release-notes.md @@ -0,0 +1 @@ +Life Science Metadata block (biomedical.tsv) was updated. "Other Design Type", "Other Factor Type", "Other Technology Type", "Other Technology Platform" boxes were added. To update the existing Life Science Metadata block run `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"`. To be able to publish one needs to reload solr: copy `schema_dv_mdb_fields.xml` and `schema_dv_mdb_copies.xml` to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory and reload the solr, for example, `http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1` diff --git a/conf/solr/7.7.2/schema_dv_mdb_copies.xml b/conf/solr/7.7.2/schema_dv_mdb_copies.xml index 0208fdf3910..080cc71ef50 100644 --- a/conf/solr/7.7.2/schema_dv_mdb_copies.xml +++ b/conf/solr/7.7.2/schema_dv_mdb_copies.xml @@ -133,9 +133,13 @@ + + + + @@ -154,4 +158,4 @@ - \ No newline at end of file + diff --git a/conf/solr/7.7.2/schema_dv_mdb_fields.xml b/conf/solr/7.7.2/schema_dv_mdb_fields.xml index 6caa7c6de69..3f844c6183c 100644 --- a/conf/solr/7.7.2/schema_dv_mdb_fields.xml +++ b/conf/solr/7.7.2/schema_dv_mdb_fields.xml @@ -133,9 +133,13 @@ + + + + @@ -154,4 +158,4 @@ - \ No newline at end of file + diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv index f45c5849845..28d59130c34 100644 --- a/scripts/api/data/metadatablocks/biomedical.tsv +++ b/scripts/api/data/metadatablocks/biomedical.tsv @@ -1,295 +1,299 @@ -#metadataBlock name dataverseAlias displayName - biomedical Life Sciences Metadata -#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id - studyDesignType Design Type Design types that are based on the overall experimental design. text 0 TRUE TRUE TRUE TRUE FALSE FALSE biomedical - studyFactorType Factor Type Factors used in the Dataset. text 1 TRUE TRUE TRUE TRUE FALSE FALSE biomedical - studyAssayOrganism Organism The taxonomic name of the organism used in the Dataset or from which the starting biological material derives. text 2 TRUE TRUE TRUE TRUE FALSE FALSE biomedical - studyAssayOtherOrganism Other Organism If Other was selected in Organism, list any other organisms that were used in this Dataset. Terms from the NCBI Taxonomy are recommended. text 3 TRUE FALSE TRUE TRUE FALSE FALSE biomedical - studyAssayMeasurementType Measurement Type A term to qualify the endpoint, or what is being measured (e.g. gene expression profiling; protein identification). text 4 TRUE TRUE TRUE TRUE FALSE FALSE biomedical - studyAssayOtherMeasurmentType Other Measurement Type If Other was selected in Measurement Type, list any other measurement types that were used. Terms from NCBO Bioportal are recommended. text 5 TRUE FALSE TRUE TRUE FALSE FALSE biomedical - studyAssayTechnologyType Technology Type A term to identify the technology used to perform the measurement (e.g. DNA microarray; mass spectrometry). text 6 TRUE TRUE TRUE TRUE FALSE FALSE biomedical - studyAssayPlatform Technology Platform The manufacturer and name of the technology platform used in the assay (e.g. Bruker AVANCE). text 7 TRUE TRUE TRUE TRUE FALSE FALSE biomedical - studyAssayCellType Cell Type The name of the cell line from which the source or sample derives. text 8 TRUE TRUE TRUE TRUE FALSE FALSE biomedical -#controlledVocabulary DatasetField Value identifier displayOrder - studyDesignType Case Control EFO_0001427 0 - studyDesignType Cross Sectional EFO_0001428 1 - studyDesignType Cohort Study OCRE100078 2 - studyDesignType Nested Case Control Design NCI_C48202 3 - studyDesignType Not Specified OTHER_DESIGN 4 - studyDesignType Parallel Group Design OBI_0500006 5 - studyDesignType Perturbation Design OBI_0001033 6 - studyDesignType Randomized Controlled Trial MESH_D016449 7 - studyDesignType Technological Design TECH_DESIGN 8 - studyFactorType Age EFO_0000246 0 - studyFactorType Biomarkers BIOMARKERS 1 - studyFactorType Cell Surface Markers CELL_SURFACE_M 2 - studyFactorType Cell Type/Cell Line EFO_0000324;EFO_0000322 3 - studyFactorType Developmental Stage EFO_0000399 4 - studyFactorType Disease State OBI_0001293 5 - studyFactorType Drug Susceptibility IDO_0000469 6 - studyFactorType Extract Molecule FBcv_0010001 7 - studyFactorType Genetic Characteristics OBI_0001404 8 - studyFactorType Immunoprecipitation Antibody OBI_0000690 9 - studyFactorType Organism OBI_0100026 10 - studyFactorType Other OTHER_FACTOR 11 - studyFactorType Passages PASSAGES_FACTOR 12 - studyFactorType Platform OBI_0000050 13 - studyFactorType Sex EFO_0000695 14 - studyFactorType Strain EFO_0005135 15 - studyFactorType Time Point EFO_0000724 16 - studyFactorType Tissue Type BTO_0001384 17 - studyFactorType Treatment Compound EFO_0000369 18 - studyFactorType Treatment Type EFO_0000727 19 - studyAssayMeasurementType cell counting ERO_0001899 0 - studyAssayMeasurementType cell sorting CHMO_0001085 1 - studyAssayMeasurementType clinical chemistry analysis OBI_0000520 2 - studyAssayMeasurementType copy number variation profiling OBI_0000537 3 - studyAssayMeasurementType DNA methylation profiling OBI_0000634 4 - studyAssayMeasurementType DNA methylation profiling (Bisulfite-Seq) OBI_0000748 5 - studyAssayMeasurementType DNA methylation profiling (MeDIP-Seq) _OBI_0000634 6 - studyAssayMeasurementType drug susceptibility _IDO_0000469 7 - studyAssayMeasurementType environmental gene survey ENV_GENE_SURVEY 8 - studyAssayMeasurementType genome sequencing ERO_0001183 9 - studyAssayMeasurementType hematology OBI_0000630 10 - studyAssayMeasurementType histology OBI_0600020 11 - studyAssayMeasurementType Histone Modification (ChIP-Seq) OBI_0002017 12 - studyAssayMeasurementType loss of heterozygosity profiling SO_0001786 13 - studyAssayMeasurementType metabolite profiling OBI_0000366 14 - studyAssayMeasurementType metagenome sequencing METAGENOME_SEQ 15 - studyAssayMeasurementType protein expression profiling OBI_0000615 16 - studyAssayMeasurementType protein identification ERO_0000346 17 - studyAssayMeasurementType protein-DNA binding site identification PROTEIN_DNA_BINDING 18 - studyAssayMeasurementType protein-protein interaction detection OBI_0000288 19 - studyAssayMeasurementType protein-RNA binding (RIP-Seq) PROTEIN_RNA_BINDING 20 - studyAssayMeasurementType SNP analysis OBI_0000435 21 - studyAssayMeasurementType targeted sequencing TARGETED_SEQ 22 - studyAssayMeasurementType transcription factor binding (ChIP-Seq) OBI_0002018 23 - studyAssayMeasurementType transcription factor binding site identification OBI_0000291 24 - studyAssayMeasurementType transcription profiling OBI_0000424 25 - studyAssayMeasurementType transcription profiling EFO_0001032 26 - studyAssayMeasurementType transcription profiling (Microarray) TRANSCRIPTION_PROF 27 - studyAssayMeasurementType transcription profiling (RNA-Seq) OBI_0001271 28 - studyAssayMeasurementType TRAP translational profiling TRAP_TRANS_PROF 29 - studyAssayMeasurementType Other OTHER_MEASUREMENT 30 - studyAssayOrganism Arabidopsis thaliana NCBITaxon_3702 0 - studyAssayOrganism Bos taurus NCBITaxon_9913 1 - studyAssayOrganism Caenorhabditis elegans NCBITaxon_6239 2 - studyAssayOrganism Chlamydomonas reinhardtii NCBITaxon_3055 3 - studyAssayOrganism Danio rerio (zebrafish) NCBITaxon_7955 4 - studyAssayOrganism Dictyostelium discoideum NCBITaxon_44689 5 - studyAssayOrganism Drosophila melanogaster NCBITaxon_7227 6 - studyAssayOrganism Escherichia coli NCBITaxon_562 7 - studyAssayOrganism Hepatitis C virus NCBITaxon_11103 8 - studyAssayOrganism Homo sapiens NCBITaxon_9606 9 - studyAssayOrganism Mus musculus NCBITaxon_10090 10 - studyAssayOrganism Mycobacterium africanum NCBITaxon_33894 11 - studyAssayOrganism Mycobacterium canetti NCBITaxon_78331 12 - studyAssayOrganism Mycobacterium tuberculosis NCBITaxon_1773 13 - studyAssayOrganism Mycoplasma pneumoniae NCBITaxon_2104 14 - studyAssayOrganism Oryza sativa NCBITaxon_4530 15 - studyAssayOrganism Plasmodium falciparum NCBITaxon_5833 16 - studyAssayOrganism Pneumocystis carinii NCBITaxon_4754 17 - studyAssayOrganism Rattus norvegicus NCBITaxon_10116 18 - studyAssayOrganism Saccharomyces cerevisiae (brewer's yeast) NCBITaxon_4932 19 - studyAssayOrganism Schizosaccharomyces pombe NCBITaxon_4896 20 - studyAssayOrganism Takifugu rubripes NCBITaxon_31033 21 - studyAssayOrganism Xenopus laevis NCBITaxon_8355 22 - studyAssayOrganism Zea mays NCBITaxon_4577 23 - studyAssayOrganism Other OTHER_TAXONOMY 24 - studyAssayTechnologyType culture based drug susceptibility testing, single concentration CULTURE_DRUG_TEST_SINGLE 0 - studyAssayTechnologyType culture based drug susceptibility testing, two concentrations CULTURE_DRUG_TEST_TWO 1 - studyAssayTechnologyType culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement) CULTURE_DRUG_TEST_THREE 2 - studyAssayTechnologyType DNA microarray OBI_0400148 3 - studyAssayTechnologyType flow cytometry OBI_0000916 4 - studyAssayTechnologyType gel electrophoresis OBI_0600053 5 - studyAssayTechnologyType mass spectrometry OBI_0000470 6 - studyAssayTechnologyType NMR spectroscopy OBI_0000623 7 - studyAssayTechnologyType nucleotide sequencing OBI_0000626 8 - studyAssayTechnologyType protein microarray OBI_0400149 9 - studyAssayTechnologyType real time PCR OBI_0000893 10 - studyAssayTechnologyType no technology required NO_TECHNOLOGY 11 - studyAssayTechnologyType Other OTHER_TECHNOLOGY 12 - studyAssayPlatform 210-MS GC Ion Trap (Varian) 210_MS_GC 0 - studyAssayPlatform 220-MS GC Ion Trap (Varian) 220_MS_GC 1 - studyAssayPlatform 225-MS GC Ion Trap (Varian) 225_MS_GC 2 - studyAssayPlatform 240-MS GC Ion Trap (Varian) 240_MS_GC 3 - studyAssayPlatform 300-MS quadrupole GC/MS (Varian) 300_MS_GCMS 4 - studyAssayPlatform 320-MS LC/MS (Varian) 320_MS_LCMS 5 - studyAssayPlatform 325-MS LC/MS (Varian) 325_MS_LCMS 6 - studyAssayPlatform 320-MS GC/MS (Varian) 500_MS_GCMS 7 - studyAssayPlatform 500-MS LC/MS (Varian) 500_MS_LCMS 8 - studyAssayPlatform 800D (Jeol) 800D 9 - studyAssayPlatform 910-MS TQ-FT (Varian) 910_MS_TQFT 10 - studyAssayPlatform 920-MS TQ-FT (Varian) 920_MS_TQFT 11 - studyAssayPlatform 3100 Mass Detector (Waters) 3100_MASS_D 12 - studyAssayPlatform 6110 Quadrupole LC/MS (Agilent) 6110_QUAD_LCMS 13 - studyAssayPlatform 6120 Quadrupole LC/MS (Agilent) 6120_QUAD_LCMS 14 - studyAssayPlatform 6130 Quadrupole LC/MS (Agilent) 6130_QUAD_LCMS 15 - studyAssayPlatform 6140 Quadrupole LC/MS (Agilent) 6140_QUAD_LCMS 16 - studyAssayPlatform 6310 Ion Trap LC/MS (Agilent) 6310_ION_LCMS 17 - studyAssayPlatform 6320 Ion Trap LC/MS (Agilent) 6320_ION_LCMS 18 - studyAssayPlatform 6330 Ion Trap LC/MS (Agilent) 6330_ION_LCMS 19 - studyAssayPlatform 6340 Ion Trap LC/MS (Agilent) 6340_ION_LCMS 20 - studyAssayPlatform 6410 Triple Quadrupole LC/MS (Agilent) 6410_TRIPLE_LCMS 21 - studyAssayPlatform 6430 Triple Quadrupole LC/MS (Agilent) 6430_TRIPLE_LCMS 22 - studyAssayPlatform 6460 Triple Quadrupole LC/MS (Agilent) 6460_TRIPLE_LCMS 23 - studyAssayPlatform 6490 Triple Quadrupole LC/MS (Agilent) 6490_TRIPLE_LCMS 24 - studyAssayPlatform 6530 Q-TOF LC/MS (Agilent) 6530_Q_TOF_LCMS 25 - studyAssayPlatform 6540 Q-TOF LC/MS (Agilent) 6540_Q_TOF_LCMS 26 - studyAssayPlatform 6210 TOF LC/MS (Agilent) 6210_Q_TOF_LCMS 27 - studyAssayPlatform 6220 TOF LC/MS (Agilent) 6220_Q_TOF_LCMS 28 - studyAssayPlatform 6230 TOF LC/MS (Agilent) 6230_Q_TOF_LCMS 29 - studyAssayPlatform 7000B Triple Quadrupole GC/MS (Agilent) 700B_TRIPLE_GCMS 30 - studyAssayPlatform AccuTO DART (Jeol) ACCUTO_DART 31 - studyAssayPlatform AccuTOF GC (Jeol) ACCUTOF_GC 32 - studyAssayPlatform AccuTOF LC (Jeol) ACCUTOF_LC 33 - studyAssayPlatform ACQUITY SQD (Waters) ACQUITY_SQD 34 - studyAssayPlatform ACQUITY TQD (Waters) ACQUITY_TQD 35 - studyAssayPlatform Agilent AGILENT 36 - studyAssayPlatform Agilent 5975E GC/MSD (Agilent) AGILENT_ 5975E_GCMSD 37 - studyAssayPlatform Agilent 5975T LTM GC/MSD (Agilent) AGILENT_5975T_LTM_GCMSD 38 - studyAssayPlatform 5975C Series GC/MSD (Agilent) 5975C_GCMSD 39 - studyAssayPlatform Affymetrix AFFYMETRIX 40 - studyAssayPlatform amaZon ETD ESI Ion Trap (Bruker) AMAZON_ETD_ESI 41 - studyAssayPlatform amaZon X ESI Ion Trap (Bruker) AMAZON_X_ESI 42 - studyAssayPlatform apex-ultra hybrid Qq-FTMS (Bruker) APEX_ULTRA_QQ_FTMS 43 - studyAssayPlatform API 2000 (AB Sciex) API_2000 44 - studyAssayPlatform API 3200 (AB Sciex) API_3200 45 - studyAssayPlatform API 3200 QTRAP (AB Sciex) API_3200_QTRAP 46 - studyAssayPlatform API 4000 (AB Sciex) API_4000 47 - studyAssayPlatform API 4000 QTRAP (AB Sciex) API_4000_QTRAP 48 - studyAssayPlatform API 5000 (AB Sciex) API_5000 49 - studyAssayPlatform API 5500 (AB Sciex) API_5500 50 - studyAssayPlatform API 5500 QTRAP (AB Sciex) API_5500_QTRAP 51 - studyAssayPlatform Applied Biosystems Group (ABI) APPLIED_BIOSYSTEMS 52 - studyAssayPlatform AQI Biosciences AQI_BIOSCIENCES 53 - studyAssayPlatform Atmospheric Pressure GC (Waters) ATMOS_GC 54 - studyAssayPlatform autoflex III MALDI-TOF MS (Bruker) AUTOFLEX_III_MALDI_TOF_MS 55 - studyAssayPlatform autoflex speed(Bruker) AUTOFLEX_SPEED 56 - studyAssayPlatform AutoSpec Premier (Waters) AUTOSPEC_PREMIER 57 - studyAssayPlatform AXIMA Mega TOF (Shimadzu) AXIMA_MEGA_TOF 58 - studyAssayPlatform AXIMA Performance MALDI TOF/TOF (Shimadzu) AXIMA_PERF_MALDI_TOF 59 - studyAssayPlatform A-10 Analyzer (Apogee) A_10_ANALYZER 60 - studyAssayPlatform A-40-MiniFCM (Apogee) A_40_MINIFCM 61 - studyAssayPlatform Bactiflow (Chemunex SA) BACTIFLOW 62 - studyAssayPlatform Base4innovation BASE4INNOVATION 63 - studyAssayPlatform BD BACTEC MGIT 320 BD_BACTEC_MGIT_320 64 - studyAssayPlatform BD BACTEC MGIT 960 BD_BACTEC_MGIT_960 65 - studyAssayPlatform BD Radiometric BACTEC 460TB BD_RADIO_BACTEC_460TB 66 - studyAssayPlatform BioNanomatrix BIONANOMATRIX 67 - studyAssayPlatform Cell Lab Quanta SC (Becman Coulter) CELL_LAB_QUANTA_SC 68 - studyAssayPlatform Clarus 560 D GC/MS (PerkinElmer) CLARUS_560_D_GCMS 69 - studyAssayPlatform Clarus 560 S GC/MS (PerkinElmer) CLARUS_560_S_GCMS 70 - studyAssayPlatform Clarus 600 GC/MS (PerkinElmer) CLARUS_600_GCMS 71 - studyAssayPlatform Complete Genomics COMPLETE_GENOMICS 72 - studyAssayPlatform Cyan (Dako Cytomation) CYAN 73 - studyAssayPlatform CyFlow ML (Partec) CYFLOW_ML 74 - studyAssayPlatform Cyow SL (Partec) CYFLOW_SL 75 - studyAssayPlatform CyFlow SL3 (Partec) CYFLOW_SL3 76 - studyAssayPlatform CytoBuoy (Cyto Buoy Inc) CYTOBUOY 77 - studyAssayPlatform CytoSence (Cyto Buoy Inc) CYTOSENCE 78 - studyAssayPlatform CytoSub (Cyto Buoy Inc) CYTOSUB 79 - studyAssayPlatform Danaher DANAHER 80 - studyAssayPlatform DFS (Thermo Scientific) DFS 81 - studyAssayPlatform Exactive(Thermo Scientific) EXACTIVE 82 - studyAssayPlatform FACS Canto (Becton Dickinson) FACS_CANTO 83 - studyAssayPlatform FACS Canto2 (Becton Dickinson) FACS_CANTO2 84 - studyAssayPlatform FACS Scan (Becton Dickinson) FACS_SCAN 85 - studyAssayPlatform FC 500 (Becman Coulter) FC_500 86 - studyAssayPlatform GCmate II GC/MS (Jeol) GCMATE_II 87 - studyAssayPlatform GCMS-QP2010 Plus (Shimadzu) GCMS_QP2010_PLUS 88 - studyAssayPlatform GCMS-QP2010S Plus (Shimadzu) GCMS_QP2010S_PLUS 89 - studyAssayPlatform GCT Premier (Waters) GCT_PREMIER 90 - studyAssayPlatform GENEQ GENEQ 91 - studyAssayPlatform Genome Corp. GENOME_CORP 92 - studyAssayPlatform GenoVoxx GENOVOXX 93 - studyAssayPlatform GnuBio GNUBIO 94 - studyAssayPlatform Guava EasyCyte Mini (Millipore) GUAVA_EASYCYTE_MINI 95 - studyAssayPlatform Guava EasyCyte Plus (Millipore) GUAVA_EASYCYTE_PLUS 96 - studyAssayPlatform Guava Personal Cell Analysis (Millipore) GUAVA_PERSONAL_CELL 97 - studyAssayPlatform Guava Personal Cell Analysis-96 (Millipore) GUAVA_PERSONAL_CELL_96 98 - studyAssayPlatform Helicos BioSciences HELICOS_BIO 99 - studyAssayPlatform Illumina ILLUMINA 100 - studyAssayPlatform Indirect proportion method on LJ medium INDIRECT_LJ_MEDIUM 101 - studyAssayPlatform Indirect proportion method on Middlebrook Agar 7H9 INDIRECT_AGAR_7H9 102 - studyAssayPlatform Indirect proportion method on Middlebrook Agar 7H10 INDIRECT_AGAR_7H10 103 - studyAssayPlatform Indirect proportion method on Middlebrook Agar 7H11 INDIRECT_AGAR_7H11 104 - studyAssayPlatform inFlux Analyzer (Cytopeia) INFLUX_ANALYZER 105 - studyAssayPlatform Intelligent Bio-Systems INTELLIGENT_BIOSYSTEMS 106 - studyAssayPlatform ITQ 700 (Thermo Scientific) ITQ_700 107 - studyAssayPlatform ITQ 900 (Thermo Scientific) ITQ_900 108 - studyAssayPlatform ITQ 1100 (Thermo Scientific) ITQ_1100 109 - studyAssayPlatform JMS-53000 SpiralTOF (Jeol) JMS_53000_SPIRAL 110 - studyAssayPlatform LaserGen LASERGEN 111 - studyAssayPlatform LCMS-2020 (Shimadzu) LCMS_2020 112 - studyAssayPlatform LCMS-2010EV (Shimadzu) LCMS_2010EV 113 - studyAssayPlatform LCMS-IT-TOF (Shimadzu) LCMS_IT_TOF 114 - studyAssayPlatform Li-Cor LI_COR 115 - studyAssayPlatform Life Tech LIFE_TECH 116 - studyAssayPlatform LightSpeed Genomics LIGHTSPEED_GENOMICS 117 - studyAssayPlatform LCT Premier XE (Waters) LCT_PREMIER_XE 118 - studyAssayPlatform LCQ Deca XP MAX (Thermo Scientific) LCQ_DECA_XP_MAX 119 - studyAssayPlatform LCQ Fleet (Thermo Scientific) LCQ_FLEET 120 - studyAssayPlatform LXQ (Thermo Scientific) LXQ_THERMO 121 - studyAssayPlatform LTQ Classic (Thermo Scientific) LTQ_CLASSIC 122 - studyAssayPlatform LTQ XL (Thermo Scientific) LTQ_XL 123 - studyAssayPlatform LTQ Velos (Thermo Scientific) LTQ_VELOS 124 - studyAssayPlatform LTQ Orbitrap Classic (Thermo Scientific) LTQ_ORBITRAP_CLASSIC 125 - studyAssayPlatform LTQ Orbitrap XL (Thermo Scientific) LTQ_ORBITRAP_XL 126 - studyAssayPlatform LTQ Orbitrap Discovery (Thermo Scientific) LTQ_ORBITRAP_DISCOVERY 127 - studyAssayPlatform LTQ Orbitrap Velos (Thermo Scientific) LTQ_ORBITRAP_VELOS 128 - studyAssayPlatform Luminex 100 (Luminex) LUMINEX_100 129 - studyAssayPlatform Luminex 200 (Luminex) LUMINEX_200 130 - studyAssayPlatform MACS Quant (Miltenyi) MACS_QUANT 131 - studyAssayPlatform MALDI SYNAPT G2 HDMS (Waters) MALDI_SYNAPT_G2_HDMS 132 - studyAssayPlatform MALDI SYNAPT G2 MS (Waters) MALDI_SYNAPT_G2_MS 133 - studyAssayPlatform MALDI SYNAPT HDMS (Waters) MALDI_SYNAPT_HDMS 134 - studyAssayPlatform MALDI SYNAPT MS (Waters) MALDI_SYNAPT_MS 135 - studyAssayPlatform MALDI micro MX (Waters) MALDI_MICROMX 136 - studyAssayPlatform maXis (Bruker) MAXIS 137 - studyAssayPlatform maXis G4 (Bruker) MAXISG4 138 - studyAssayPlatform microflex LT MALDI-TOF MS (Bruker) MICROFLEX_LT_MALDI_TOF_MS 139 - studyAssayPlatform microflex LRF MALDI-TOF MS (Bruker) MICROFLEX_LRF_MALDI_TOF_MS 140 - studyAssayPlatform microflex III MALDI-TOF MS (Bruker) MICROFLEX_III_TOF_MS 141 - studyAssayPlatform micrOTOF II ESI TOF (Bruker) MICROTOF_II_ESI_TOF 142 - studyAssayPlatform micrOTOF-Q II ESI-Qq-TOF (Bruker) MICROTOF_Q_II_ESI_QQ_TOF 143 - studyAssayPlatform microplate Alamar Blue (resazurin) colorimetric method MICROPLATE_ALAMAR_BLUE_COLORIMETRIC 144 - studyAssayPlatform Mstation (Jeol) MSTATION 145 - studyAssayPlatform MSQ Plus (Thermo Scientific) MSQ_PLUS 146 - studyAssayPlatform NABsys NABSYS 147 - studyAssayPlatform Nanophotonics Biosciences NANOPHOTONICS_BIOSCIENCES 148 - studyAssayPlatform Network Biosystems NETWORK_BIOSYSTEMS 149 - studyAssayPlatform Nimblegen NIMBLEGEN 150 - studyAssayPlatform Oxford Nanopore Technologies OXFORD_NANOPORE_TECHNOLOGIES 151 - studyAssayPlatform Pacific Biosciences PACIFIC_BIOSCIENCES 152 - studyAssayPlatform Population Genetics Technologies POPULATION_GENETICS_TECHNOLOGIES 153 - studyAssayPlatform Q1000GC UltraQuad (Jeol) Q1000GC_ULTRAQUAD 154 - studyAssayPlatform Quattro micro API (Waters) QUATTRO_MICRO_API 155 - studyAssayPlatform Quattro micro GC (Waters) QUATTRO_MICRO_GC 156 - studyAssayPlatform Quattro Premier XE (Waters) QUATTRO_PREMIER_XE 157 - studyAssayPlatform QSTAR (AB Sciex) QSTAR 158 - studyAssayPlatform Reveo REVEO 159 - studyAssayPlatform Roche ROCHE 160 - studyAssayPlatform Seirad SEIRAD 161 - studyAssayPlatform solariX hybrid Qq-FTMS (Bruker) SOLARIX_HYBRID_QQ_FTMS 162 - studyAssayPlatform Somacount (Bently Instruments) SOMACOUNT 163 - studyAssayPlatform SomaScope (Bently Instruments) SOMASCOPE 164 - studyAssayPlatform SYNAPT G2 HDMS (Waters) SYNAPT_G2_HDMS 165 - studyAssayPlatform SYNAPT G2 MS (Waters) SYNAPT_G2_MS 166 - studyAssayPlatform SYNAPT HDMS (Waters) SYNAPT_HDMS 167 - studyAssayPlatform SYNAPT MS (Waters) SYNAPT_MS 168 - studyAssayPlatform TripleTOF 5600 (AB Sciex) TRIPLETOF_5600 169 - studyAssayPlatform TSQ Quantum Ultra (Thermo Scientific) TSQ_QUANTUM_ULTRA 170 - studyAssayPlatform TSQ Quantum Access (Thermo Scientific) TSQ_QUANTUM_ACCESS 171 - studyAssayPlatform TSQ Quantum Access MAX (Thermo Scientific) TSQ_QUANTUM_ACCESS_MAX 172 - studyAssayPlatform TSQ Quantum Discovery MAX (Thermo Scientific) TSQ_QUANTUM_DISCOVERY_MAX 173 - studyAssayPlatform TSQ Quantum GC (Thermo Scientific) TSQ_QUANTUM_GC 174 - studyAssayPlatform TSQ Quantum XLS (Thermo Scientific) TSQ_QUANTUM_XLS 175 - studyAssayPlatform TSQ Vantage (Thermo Scientific) TSQ_VANTAGE 176 - studyAssayPlatform ultrafleXtreme MALDI-TOF MS (Bruker) ULTRAFLEXTREME_MALDI_TOF_MS 177 - studyAssayPlatform VisiGen Biotechnologies VISIGEN_BIO 178 - studyAssayPlatform Xevo G2 QTOF (Waters) XEVO_G2_QTOF 179 - studyAssayPlatform Xevo QTof MS (Waters) XEVO_QTOF_MS 180 - studyAssayPlatform Xevo TQ MS (Waters) XEVO_TQ_MS 181 - studyAssayPlatform Xevo TQ-S (Waters) XEVO_TQ_S 182 +#metadataBlock name dataverseAlias displayName + biomedical Life Sciences Metadata +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id + studyDesignType Design Type Design types that are based on the overall experimental design. text 0 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyOtherDesignType Other Design Type If Other was selected in Design Type, list any other design types that were used in this Dataset. text 1 TRUE FALSE TRUE TRUE FALSE FALSE biomedical + studyFactorType Factor Type Factors used in the Dataset. text 2 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyOtherFactorType Other Factor Type If Other was selected in Factor Type, list any other factor types that were used in this Dataset. text 3 TRUE FALSE TRUE TRUE FALSE FALSE biomedical + studyAssayOrganism Organism The taxonomic name of the organism used in the Dataset or from which the starting biological material derives. text 4 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyAssayOtherOrganism Other Organism If Other was selected in Organism, list any other organisms that were used in this Dataset. Terms from the NCBI Taxonomy are recommended. text 5 TRUE FALSE TRUE TRUE FALSE FALSE biomedical + studyAssayMeasurementType Measurement Type A term to qualify the endpoint, or what is being measured (e.g. gene expression profiling; protein identification). text 6 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyAssayOtherMeasurmentType Other Measurement Type If Other was selected in Measurement Type, list any other measurement types that were used. Terms from NCBO Bioportal are recommended. text 7 TRUE FALSE TRUE TRUE FALSE FALSE biomedical + studyAssayTechnologyType Technology Type A term to identify the technology used to perform the measurement (e.g. DNA microarray; mass spectrometry). text 8 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyAssayOtherTechnologyType Other Technology Type If Other was selected in Technology Type, list any other technology types that were used in this Dataset. text 9 TRUE FALSE TRUE TRUE FALSE FALSE biomedical + studyAssayPlatform Technology Platform The manufacturer and name of the technology platform used in the assay (e.g. Bruker AVANCE). text 10 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyAssayOtherPlatform Other Technology Platform If Other was selected in Technology Platform, list any other technology platforms that were used in this Dataset. text 11 TRUE FALSE TRUE TRUE FALSE FALSE biomedical + studyAssayCellType Cell Type The name of the cell line from which the source or sample derives. text 12 TRUE TRUE TRUE TRUE FALSE FALSE biomedical +#controlledVocabulary DatasetField Value identifier displayOrder + studyDesignType Case Control EFO_0001427 0 + studyDesignType Cross Sectional EFO_0001428 1 + studyDesignType Cohort Study OCRE100078 2 + studyDesignType Nested Case Control Design NCI_C48202 3 + studyDesignType Not Specified NOT_SPECIFIED 4 + studyDesignType Parallel Group Design OBI_0500006 5 + studyDesignType Perturbation Design OBI_0001033 6 + studyDesignType Randomized Controlled Trial MESH_D016449 7 + studyDesignType Technological Design TECH_DESIGN 8 + studyDesignType Other OTHER_DESIGN 9 + studyFactorType Age EFO_0000246 0 + studyFactorType Biomarkers BIOMARKERS 1 + studyFactorType Cell Surface Markers CELL_SURFACE_M 2 + studyFactorType Cell Type/Cell Line EFO_0000324;EFO_0000322 3 + studyFactorType Developmental Stage EFO_0000399 4 + studyFactorType Disease State OBI_0001293 5 + studyFactorType Drug Susceptibility IDO_0000469 6 + studyFactorType Extract Molecule FBcv_0010001 7 + studyFactorType Genetic Characteristics OBI_0001404 8 + studyFactorType Immunoprecipitation Antibody OBI_0000690 9 + studyFactorType Organism OBI_0100026 10 + studyFactorType Passages PASSAGES_FACTOR 11 + studyFactorType Platform OBI_0000050 12 + studyFactorType Sex EFO_0000695 13 + studyFactorType Strain EFO_0005135 14 + studyFactorType Time Point EFO_0000724 15 + studyFactorType Tissue Type BTO_0001384 16 + studyFactorType Treatment Compound EFO_0000369 17 + studyFactorType Treatment Type EFO_0000727 18 + studyFactorType Other OTHER_FACTOR 19 + studyAssayMeasurementType cell sorting CHMO_0001085 1 + studyAssayMeasurementType clinical chemistry analysis OBI_0000520 2 + studyAssayMeasurementType copy number variation profiling OBI_0000537 3 + studyAssayMeasurementType DNA methylation profiling OBI_0000634 4 + studyAssayMeasurementType DNA methylation profiling (Bisulfite-Seq) OBI_0000748 5 + studyAssayMeasurementType DNA methylation profiling (MeDIP-Seq) _OBI_0000634 6 + studyAssayMeasurementType drug susceptibility _IDO_0000469 7 + studyAssayMeasurementType environmental gene survey ENV_GENE_SURVEY 8 + studyAssayMeasurementType genome sequencing ERO_0001183 9 + studyAssayMeasurementType hematology OBI_0000630 10 + studyAssayMeasurementType histology OBI_0600020 11 + studyAssayMeasurementType Histone Modification (ChIP-Seq) OBI_0002017 12 + studyAssayMeasurementType loss of heterozygosity profiling SO_0001786 13 + studyAssayMeasurementType metabolite profiling OBI_0000366 14 + studyAssayMeasurementType metagenome sequencing METAGENOME_SEQ 15 + studyAssayMeasurementType protein expression profiling OBI_0000615 16 + studyAssayMeasurementType protein identification ERO_0000346 17 + studyAssayMeasurementType protein-DNA binding site identification PROTEIN_DNA_BINDING 18 + studyAssayMeasurementType protein-protein interaction detection OBI_0000288 19 + studyAssayMeasurementType protein-RNA binding (RIP-Seq) PROTEIN_RNA_BINDING 20 + studyAssayMeasurementType SNP analysis OBI_0000435 21 + studyAssayMeasurementType targeted sequencing TARGETED_SEQ 22 + studyAssayMeasurementType transcription factor binding (ChIP-Seq) OBI_0002018 23 + studyAssayMeasurementType transcription factor binding site identification OBI_0000291 24 + studyAssayMeasurementType transcription profiling OBI_0000424 25 + studyAssayMeasurementType transcription profiling EFO_0001032 26 + studyAssayMeasurementType transcription profiling (Microarray) TRANSCRIPTION_PROF 27 + studyAssayMeasurementType transcription profiling (RNA-Seq) OBI_0001271 28 + studyAssayMeasurementType TRAP translational profiling TRAP_TRANS_PROF 29 + studyAssayMeasurementType Other OTHER_MEASUREMENT 30 + studyAssayOrganism Arabidopsis thaliana NCBITaxon_3702 0 + studyAssayOrganism Bos taurus NCBITaxon_9913 1 + studyAssayOrganism Caenorhabditis elegans NCBITaxon_6239 2 + studyAssayOrganism Chlamydomonas reinhardtii NCBITaxon_3055 3 + studyAssayOrganism Danio rerio (zebrafish) NCBITaxon_7955 4 + studyAssayOrganism Dictyostelium discoideum NCBITaxon_44689 5 + studyAssayOrganism Drosophila melanogaster NCBITaxon_7227 6 + studyAssayOrganism Escherichia coli NCBITaxon_562 7 + studyAssayOrganism Hepatitis C virus NCBITaxon_11103 8 + studyAssayOrganism Homo sapiens NCBITaxon_9606 9 + studyAssayOrganism Mus musculus NCBITaxon_10090 10 + studyAssayOrganism Mycobacterium africanum NCBITaxon_33894 11 + studyAssayOrganism Mycobacterium canetti NCBITaxon_78331 12 + studyAssayOrganism Mycobacterium tuberculosis NCBITaxon_1773 13 + studyAssayOrganism Mycoplasma pneumoniae NCBITaxon_2104 14 + studyAssayOrganism Oryza sativa NCBITaxon_4530 15 + studyAssayOrganism Plasmodium falciparum NCBITaxon_5833 16 + studyAssayOrganism Pneumocystis carinii NCBITaxon_4754 17 + studyAssayOrganism Rattus norvegicus NCBITaxon_10116 18 + studyAssayOrganism Saccharomyces cerevisiae (brewer's yeast) NCBITaxon_4932 19 + studyAssayOrganism Schizosaccharomyces pombe NCBITaxon_4896 20 + studyAssayOrganism Takifugu rubripes NCBITaxon_31033 21 + studyAssayOrganism Xenopus laevis NCBITaxon_8355 22 + studyAssayOrganism Zea mays NCBITaxon_4577 23 + studyAssayOrganism Other OTHER_TAXONOMY 24 + studyAssayTechnologyType culture based drug susceptibility testing, single concentration CULTURE_DRUG_TEST_SINGLE 0 + studyAssayTechnologyType culture based drug susceptibility testing, two concentrations CULTURE_DRUG_TEST_TWO 1 + studyAssayTechnologyType culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement) CULTURE_DRUG_TEST_THREE 2 + studyAssayTechnologyType DNA microarray OBI_0400148 3 + studyAssayTechnologyType flow cytometry OBI_0000916 4 + studyAssayTechnologyType gel electrophoresis OBI_0600053 5 + studyAssayTechnologyType mass spectrometry OBI_0000470 6 + studyAssayTechnologyType NMR spectroscopy OBI_0000623 7 + studyAssayTechnologyType nucleotide sequencing OBI_0000626 8 + studyAssayTechnologyType protein microarray OBI_0400149 9 + studyAssayTechnologyType real time PCR OBI_0000893 10 + studyAssayTechnologyType no technology required NO_TECHNOLOGY 11 + studyAssayTechnologyType Other OTHER_TECHNOLOGY 12 + studyAssayPlatform 210-MS GC Ion Trap (Varian) 210_MS_GC 0 + studyAssayPlatform 220-MS GC Ion Trap (Varian) 220_MS_GC 1 + studyAssayPlatform 225-MS GC Ion Trap (Varian) 225_MS_GC 2 + studyAssayPlatform 240-MS GC Ion Trap (Varian) 240_MS_GC 3 + studyAssayPlatform 300-MS quadrupole GC/MS (Varian) 300_MS_GCMS 4 + studyAssayPlatform 320-MS LC/MS (Varian) 320_MS_LCMS 5 + studyAssayPlatform 325-MS LC/MS (Varian) 325_MS_LCMS 6 + studyAssayPlatform 320-MS GC/MS (Varian) 500_MS_GCMS 7 + studyAssayPlatform 500-MS LC/MS (Varian) 500_MS_LCMS 8 + studyAssayPlatform 800D (Jeol) 800D 9 + studyAssayPlatform 910-MS TQ-FT (Varian) 910_MS_TQFT 10 + studyAssayPlatform 920-MS TQ-FT (Varian) 920_MS_TQFT 11 + studyAssayPlatform 3100 Mass Detector (Waters) 3100_MASS_D 12 + studyAssayPlatform 6110 Quadrupole LC/MS (Agilent) 6110_QUAD_LCMS 13 + studyAssayPlatform 6120 Quadrupole LC/MS (Agilent) 6120_QUAD_LCMS 14 + studyAssayPlatform 6130 Quadrupole LC/MS (Agilent) 6130_QUAD_LCMS 15 + studyAssayPlatform 6140 Quadrupole LC/MS (Agilent) 6140_QUAD_LCMS 16 + studyAssayPlatform 6310 Ion Trap LC/MS (Agilent) 6310_ION_LCMS 17 + studyAssayPlatform 6320 Ion Trap LC/MS (Agilent) 6320_ION_LCMS 18 + studyAssayPlatform 6330 Ion Trap LC/MS (Agilent) 6330_ION_LCMS 19 + studyAssayPlatform 6340 Ion Trap LC/MS (Agilent) 6340_ION_LCMS 20 + studyAssayPlatform 6410 Triple Quadrupole LC/MS (Agilent) 6410_TRIPLE_LCMS 21 + studyAssayPlatform 6430 Triple Quadrupole LC/MS (Agilent) 6430_TRIPLE_LCMS 22 + studyAssayPlatform 6460 Triple Quadrupole LC/MS (Agilent) 6460_TRIPLE_LCMS 23 + studyAssayPlatform 6490 Triple Quadrupole LC/MS (Agilent) 6490_TRIPLE_LCMS 24 + studyAssayPlatform 6530 Q-TOF LC/MS (Agilent) 6530_Q_TOF_LCMS 25 + studyAssayPlatform 6540 Q-TOF LC/MS (Agilent) 6540_Q_TOF_LCMS 26 + studyAssayPlatform 6210 TOF LC/MS (Agilent) 6210_Q_TOF_LCMS 27 + studyAssayPlatform 6220 TOF LC/MS (Agilent) 6220_Q_TOF_LCMS 28 + studyAssayPlatform 6230 TOF LC/MS (Agilent) 6230_Q_TOF_LCMS 29 + studyAssayPlatform 7000B Triple Quadrupole GC/MS (Agilent) 700B_TRIPLE_GCMS 30 + studyAssayPlatform AccuTO DART (Jeol) ACCUTO_DART 31 + studyAssayPlatform AccuTOF GC (Jeol) ACCUTOF_GC 32 + studyAssayPlatform AccuTOF LC (Jeol) ACCUTOF_LC 33 + studyAssayPlatform ACQUITY SQD (Waters) ACQUITY_SQD 34 + studyAssayPlatform ACQUITY TQD (Waters) ACQUITY_TQD 35 + studyAssayPlatform Agilent AGILENT 36 + studyAssayPlatform Agilent 5975E GC/MSD (Agilent) AGILENT_ 5975E_GCMSD 37 + studyAssayPlatform Agilent 5975T LTM GC/MSD (Agilent) AGILENT_5975T_LTM_GCMSD 38 + studyAssayPlatform 5975C Series GC/MSD (Agilent) 5975C_GCMSD 39 + studyAssayPlatform Affymetrix AFFYMETRIX 40 + studyAssayPlatform amaZon ETD ESI Ion Trap (Bruker) AMAZON_ETD_ESI 41 + studyAssayPlatform amaZon X ESI Ion Trap (Bruker) AMAZON_X_ESI 42 + studyAssayPlatform apex-ultra hybrid Qq-FTMS (Bruker) APEX_ULTRA_QQ_FTMS 43 + studyAssayPlatform API 2000 (AB Sciex) API_2000 44 + studyAssayPlatform API 3200 (AB Sciex) API_3200 45 + studyAssayPlatform API 3200 QTRAP (AB Sciex) API_3200_QTRAP 46 + studyAssayPlatform API 4000 (AB Sciex) API_4000 47 + studyAssayPlatform API 4000 QTRAP (AB Sciex) API_4000_QTRAP 48 + studyAssayPlatform API 5000 (AB Sciex) API_5000 49 + studyAssayPlatform API 5500 (AB Sciex) API_5500 50 + studyAssayPlatform API 5500 QTRAP (AB Sciex) API_5500_QTRAP 51 + studyAssayPlatform Applied Biosystems Group (ABI) APPLIED_BIOSYSTEMS 52 + studyAssayPlatform AQI Biosciences AQI_BIOSCIENCES 53 + studyAssayPlatform Atmospheric Pressure GC (Waters) ATMOS_GC 54 + studyAssayPlatform autoflex III MALDI-TOF MS (Bruker) AUTOFLEX_III_MALDI_TOF_MS 55 + studyAssayPlatform autoflex speed(Bruker) AUTOFLEX_SPEED 56 + studyAssayPlatform AutoSpec Premier (Waters) AUTOSPEC_PREMIER 57 + studyAssayPlatform AXIMA Mega TOF (Shimadzu) AXIMA_MEGA_TOF 58 + studyAssayPlatform AXIMA Performance MALDI TOF/TOF (Shimadzu) AXIMA_PERF_MALDI_TOF 59 + studyAssayPlatform A-10 Analyzer (Apogee) A_10_ANALYZER 60 + studyAssayPlatform A-40-MiniFCM (Apogee) A_40_MINIFCM 61 + studyAssayPlatform Bactiflow (Chemunex SA) BACTIFLOW 62 + studyAssayPlatform Base4innovation BASE4INNOVATION 63 + studyAssayPlatform BD BACTEC MGIT 320 BD_BACTEC_MGIT_320 64 + studyAssayPlatform BD BACTEC MGIT 960 BD_BACTEC_MGIT_960 65 + studyAssayPlatform BD Radiometric BACTEC 460TB BD_RADIO_BACTEC_460TB 66 + studyAssayPlatform BioNanomatrix BIONANOMATRIX 67 + studyAssayPlatform Cell Lab Quanta SC (Becman Coulter) CELL_LAB_QUANTA_SC 68 + studyAssayPlatform Clarus 560 D GC/MS (PerkinElmer) CLARUS_560_D_GCMS 69 + studyAssayPlatform Clarus 560 S GC/MS (PerkinElmer) CLARUS_560_S_GCMS 70 + studyAssayPlatform Clarus 600 GC/MS (PerkinElmer) CLARUS_600_GCMS 71 + studyAssayPlatform Complete Genomics COMPLETE_GENOMICS 72 + studyAssayPlatform Cyan (Dako Cytomation) CYAN 73 + studyAssayPlatform CyFlow ML (Partec) CYFLOW_ML 74 + studyAssayPlatform Cyow SL (Partec) CYFLOW_SL 75 + studyAssayPlatform CyFlow SL3 (Partec) CYFLOW_SL3 76 + studyAssayPlatform CytoBuoy (Cyto Buoy Inc) CYTOBUOY 77 + studyAssayPlatform CytoSence (Cyto Buoy Inc) CYTOSENCE 78 + studyAssayPlatform CytoSub (Cyto Buoy Inc) CYTOSUB 79 + studyAssayPlatform Danaher DANAHER 80 + studyAssayPlatform DFS (Thermo Scientific) DFS 81 + studyAssayPlatform Exactive(Thermo Scientific) EXACTIVE 82 + studyAssayPlatform FACS Canto (Becton Dickinson) FACS_CANTO 83 + studyAssayPlatform FACS Canto2 (Becton Dickinson) FACS_CANTO2 84 + studyAssayPlatform FACS Scan (Becton Dickinson) FACS_SCAN 85 + studyAssayPlatform FC 500 (Becman Coulter) FC_500 86 + studyAssayPlatform GCmate II GC/MS (Jeol) GCMATE_II 87 + studyAssayPlatform GCMS-QP2010 Plus (Shimadzu) GCMS_QP2010_PLUS 88 + studyAssayPlatform GCMS-QP2010S Plus (Shimadzu) GCMS_QP2010S_PLUS 89 + studyAssayPlatform GCT Premier (Waters) GCT_PREMIER 90 + studyAssayPlatform GENEQ GENEQ 91 + studyAssayPlatform Genome Corp. GENOME_CORP 92 + studyAssayPlatform GenoVoxx GENOVOXX 93 + studyAssayPlatform GnuBio GNUBIO 94 + studyAssayPlatform Guava EasyCyte Mini (Millipore) GUAVA_EASYCYTE_MINI 95 + studyAssayPlatform Guava EasyCyte Plus (Millipore) GUAVA_EASYCYTE_PLUS 96 + studyAssayPlatform Guava Personal Cell Analysis (Millipore) GUAVA_PERSONAL_CELL 97 + studyAssayPlatform Guava Personal Cell Analysis-96 (Millipore) GUAVA_PERSONAL_CELL_96 98 + studyAssayPlatform Helicos BioSciences HELICOS_BIO 99 + studyAssayPlatform Illumina ILLUMINA 100 + studyAssayPlatform Indirect proportion method on LJ medium INDIRECT_LJ_MEDIUM 101 + studyAssayPlatform Indirect proportion method on Middlebrook Agar 7H9 INDIRECT_AGAR_7H9 102 + studyAssayPlatform Indirect proportion method on Middlebrook Agar 7H10 INDIRECT_AGAR_7H10 103 + studyAssayPlatform Indirect proportion method on Middlebrook Agar 7H11 INDIRECT_AGAR_7H11 104 + studyAssayPlatform inFlux Analyzer (Cytopeia) INFLUX_ANALYZER 105 + studyAssayPlatform Intelligent Bio-Systems INTELLIGENT_BIOSYSTEMS 106 + studyAssayPlatform ITQ 700 (Thermo Scientific) ITQ_700 107 + studyAssayPlatform ITQ 900 (Thermo Scientific) ITQ_900 108 + studyAssayPlatform ITQ 1100 (Thermo Scientific) ITQ_1100 109 + studyAssayPlatform JMS-53000 SpiralTOF (Jeol) JMS_53000_SPIRAL 110 + studyAssayPlatform LaserGen LASERGEN 111 + studyAssayPlatform LCMS-2020 (Shimadzu) LCMS_2020 112 + studyAssayPlatform LCMS-2010EV (Shimadzu) LCMS_2010EV 113 + studyAssayPlatform LCMS-IT-TOF (Shimadzu) LCMS_IT_TOF 114 + studyAssayPlatform Li-Cor LI_COR 115 + studyAssayPlatform Life Tech LIFE_TECH 116 + studyAssayPlatform LightSpeed Genomics LIGHTSPEED_GENOMICS 117 + studyAssayPlatform LCT Premier XE (Waters) LCT_PREMIER_XE 118 + studyAssayPlatform LCQ Deca XP MAX (Thermo Scientific) LCQ_DECA_XP_MAX 119 + studyAssayPlatform LCQ Fleet (Thermo Scientific) LCQ_FLEET 120 + studyAssayPlatform LXQ (Thermo Scientific) LXQ_THERMO 121 + studyAssayPlatform LTQ Classic (Thermo Scientific) LTQ_CLASSIC 122 + studyAssayPlatform LTQ XL (Thermo Scientific) LTQ_XL 123 + studyAssayPlatform LTQ Velos (Thermo Scientific) LTQ_VELOS 124 + studyAssayPlatform LTQ Orbitrap Classic (Thermo Scientific) LTQ_ORBITRAP_CLASSIC 125 + studyAssayPlatform LTQ Orbitrap XL (Thermo Scientific) LTQ_ORBITRAP_XL 126 + studyAssayPlatform LTQ Orbitrap Discovery (Thermo Scientific) LTQ_ORBITRAP_DISCOVERY 127 + studyAssayPlatform LTQ Orbitrap Velos (Thermo Scientific) LTQ_ORBITRAP_VELOS 128 + studyAssayPlatform Luminex 100 (Luminex) LUMINEX_100 129 + studyAssayPlatform Luminex 200 (Luminex) LUMINEX_200 130 + studyAssayPlatform MACS Quant (Miltenyi) MACS_QUANT 131 + studyAssayPlatform MALDI SYNAPT G2 HDMS (Waters) MALDI_SYNAPT_G2_HDMS 132 + studyAssayPlatform MALDI SYNAPT G2 MS (Waters) MALDI_SYNAPT_G2_MS 133 + studyAssayPlatform MALDI SYNAPT HDMS (Waters) MALDI_SYNAPT_HDMS 134 + studyAssayPlatform MALDI SYNAPT MS (Waters) MALDI_SYNAPT_MS 135 + studyAssayPlatform MALDI micro MX (Waters) MALDI_MICROMX 136 + studyAssayPlatform maXis (Bruker) MAXIS 137 + studyAssayPlatform maXis G4 (Bruker) MAXISG4 138 + studyAssayPlatform microflex LT MALDI-TOF MS (Bruker) MICROFLEX_LT_MALDI_TOF_MS 139 + studyAssayPlatform microflex LRF MALDI-TOF MS (Bruker) MICROFLEX_LRF_MALDI_TOF_MS 140 + studyAssayPlatform microflex III MALDI-TOF MS (Bruker) MICROFLEX_III_TOF_MS 141 + studyAssayPlatform micrOTOF II ESI TOF (Bruker) MICROTOF_II_ESI_TOF 142 + studyAssayPlatform micrOTOF-Q II ESI-Qq-TOF (Bruker) MICROTOF_Q_II_ESI_QQ_TOF 143 + studyAssayPlatform microplate Alamar Blue (resazurin) colorimetric method MICROPLATE_ALAMAR_BLUE_COLORIMETRIC 144 + studyAssayPlatform Mstation (Jeol) MSTATION 145 + studyAssayPlatform MSQ Plus (Thermo Scientific) MSQ_PLUS 146 + studyAssayPlatform NABsys NABSYS 147 + studyAssayPlatform Nanophotonics Biosciences NANOPHOTONICS_BIOSCIENCES 148 + studyAssayPlatform Network Biosystems NETWORK_BIOSYSTEMS 149 + studyAssayPlatform Nimblegen NIMBLEGEN 150 + studyAssayPlatform Oxford Nanopore Technologies OXFORD_NANOPORE_TECHNOLOGIES 151 + studyAssayPlatform Pacific Biosciences PACIFIC_BIOSCIENCES 152 + studyAssayPlatform Population Genetics Technologies POPULATION_GENETICS_TECHNOLOGIES 153 + studyAssayPlatform Q1000GC UltraQuad (Jeol) Q1000GC_ULTRAQUAD 154 + studyAssayPlatform Quattro micro API (Waters) QUATTRO_MICRO_API 155 + studyAssayPlatform Quattro micro GC (Waters) QUATTRO_MICRO_GC 156 + studyAssayPlatform Quattro Premier XE (Waters) QUATTRO_PREMIER_XE 157 + studyAssayPlatform QSTAR (AB Sciex) QSTAR 158 + studyAssayPlatform Reveo REVEO 159 + studyAssayPlatform Roche ROCHE 160 + studyAssayPlatform Seirad SEIRAD 161 + studyAssayPlatform solariX hybrid Qq-FTMS (Bruker) SOLARIX_HYBRID_QQ_FTMS 162 + studyAssayPlatform Somacount (Bently Instruments) SOMACOUNT 163 + studyAssayPlatform SomaScope (Bently Instruments) SOMASCOPE 164 + studyAssayPlatform SYNAPT G2 HDMS (Waters) SYNAPT_G2_HDMS 165 + studyAssayPlatform SYNAPT G2 MS (Waters) SYNAPT_G2_MS 166 + studyAssayPlatform SYNAPT HDMS (Waters) SYNAPT_HDMS 167 + studyAssayPlatform SYNAPT MS (Waters) SYNAPT_MS 168 + studyAssayPlatform TripleTOF 5600 (AB Sciex) TRIPLETOF_5600 169 + studyAssayPlatform TSQ Quantum Ultra (Thermo Scientific) TSQ_QUANTUM_ULTRA 170 + studyAssayPlatform TSQ Quantum Access (Thermo Scientific) TSQ_QUANTUM_ACCESS 171 + studyAssayPlatform TSQ Quantum Access MAX (Thermo Scientific) TSQ_QUANTUM_ACCESS_MAX 172 + studyAssayPlatform TSQ Quantum Discovery MAX (Thermo Scientific) TSQ_QUANTUM_DISCOVERY_MAX 173 + studyAssayPlatform TSQ Quantum GC (Thermo Scientific) TSQ_QUANTUM_GC 174 + studyAssayPlatform TSQ Quantum XLS (Thermo Scientific) TSQ_QUANTUM_XLS 175 + studyAssayPlatform TSQ Vantage (Thermo Scientific) TSQ_VANTAGE 176 + studyAssayPlatform ultrafleXtreme MALDI-TOF MS (Bruker) ULTRAFLEXTREME_MALDI_TOF_MS 177 + studyAssayPlatform VisiGen Biotechnologies VISIGEN_BIO 178 + studyAssayPlatform Xevo G2 QTOF (Waters) XEVO_G2_QTOF 179 + studyAssayPlatform Xevo QTof MS (Waters) XEVO_QTOF_MS 180 + studyAssayPlatform Xevo TQ MS (Waters) XEVO_TQ_MS 181 + studyAssayPlatform Xevo TQ-S (Waters) XEVO_TQ_S 182 studyAssayPlatform Other OTHER_PLATFORM 183 \ No newline at end of file diff --git a/src/main/java/propertyFiles/biomedical.properties b/src/main/java/propertyFiles/biomedical.properties index c3fd3f81bc7..723a4ac2f40 100644 --- a/src/main/java/propertyFiles/biomedical.properties +++ b/src/main/java/propertyFiles/biomedical.properties @@ -1,31 +1,43 @@ metadatablock.name=biomedical metadatablock.displayName=Life Sciences Metadata datasetfieldtype.studyDesignType.title=Design Type +datasetfieldtype.studyOtherDesignType.title=Other Design Type datasetfieldtype.studyFactorType.title=Factor Type +datasetfieldtype.studyOtherFactorType.title=Other Factor Type datasetfieldtype.studyAssayOrganism.title=Organism datasetfieldtype.studyAssayOtherOrganism.title=Other Organism datasetfieldtype.studyAssayMeasurementType.title=Measurement Type datasetfieldtype.studyAssayOtherMeasurmentType.title=Other Measurement Type datasetfieldtype.studyAssayTechnologyType.title=Technology Type +datasetfieldtype.studyAssayOtherTechnologyType.title=Other Technology Type datasetfieldtype.studyAssayPlatform.title=Technology Platform +datasetfieldtype.studyAssayOtherPlatform.title=Other Technology Platform datasetfieldtype.studyAssayCellType.title=Cell Type datasetfieldtype.studyDesignType.description=Design types that are based on the overall experimental design. +datasetfieldtype.studyOtherDesignType.description=If Other was selected in Design Type, list any other design types that were used in this Dataset. datasetfieldtype.studyFactorType.description=Factors used in the Dataset. +datasetfieldtype.studyOtherFactorType.description=If Other was selected in Factor Type, list any other factor types that were used in this Dataset. datasetfieldtype.studyAssayOrganism.description=The taxonomic name of the organism used in the Dataset or from which the starting biological material derives. datasetfieldtype.studyAssayOtherOrganism.description=If Other was selected in Organism, list any other organisms that were used in this Dataset. Terms from the NCBI Taxonomy are recommended. datasetfieldtype.studyAssayMeasurementType.description=A term to qualify the endpoint, or what is being measured (e.g. gene expression profiling; protein identification). datasetfieldtype.studyAssayOtherMeasurmentType.description=If Other was selected in Measurement Type, list any other measurement types that were used. Terms from NCBO Bioportal are recommended. datasetfieldtype.studyAssayTechnologyType.description=A term to identify the technology used to perform the measurement (e.g. DNA microarray; mass spectrometry). +datasetfieldtype.studyAssayOtherTechnologyType.description=If Other was selected in Technology Type, list any other technology types that were used in this Dataset. datasetfieldtype.studyAssayPlatform.description=The manufacturer and name of the technology platform used in the assay (e.g. Bruker AVANCE). +datasetfieldtype.studyAssayOtherPlatform.description=If Other was selected in Technology Platform, list any other technology platforms that were used in this Dataset. datasetfieldtype.studyAssayCellType.description=The name of the cell line from which the source or sample derives. datasetfieldtype.studyDesignType.watermark= +datasetfieldtype.studyOtherDesignType.watermark= datasetfieldtype.studyFactorType.watermark= +datasetfieldtype.studyOtherFactorType.watermark= datasetfieldtype.studyAssayOrganism.watermark= datasetfieldtype.studyAssayOtherOrganism.watermark= datasetfieldtype.studyAssayMeasurementType.watermark= datasetfieldtype.studyAssayOtherMeasurmentType.watermark= datasetfieldtype.studyAssayTechnologyType.watermark= +datasetfieldtype.studyAssayOtherTechnologyType.watermark= datasetfieldtype.studyAssayPlatform.watermark= +datasetfieldtype.studyAssayOtherPlatform.watermark= datasetfieldtype.studyAssayCellType.watermark= controlledvocabulary.studyDesignType.case_control=Case Control controlledvocabulary.studyDesignType.cross_sectional=Cross Sectional @@ -36,6 +48,7 @@ controlledvocabulary.studyDesignType.parallel_group_design=Parallel Group Design controlledvocabulary.studyDesignType.perturbation_design=Perturbation Design controlledvocabulary.studyDesignType.randomized_controlled_trial=Randomized Controlled Trial controlledvocabulary.studyDesignType.technological_design=Technological Design +controlledvocabulary.studyDesignType.other=Other controlledvocabulary.studyFactorType.age=Age controlledvocabulary.studyFactorType.biomarkers=Biomarkers controlledvocabulary.studyFactorType.cell_surface_markers=Cell Surface Markers @@ -308,4 +321,4 @@ controlledvocabulary.studyAssayPlatform.xevo_g2_qtof_(waters)=Xevo G2 QTOF (Wate controlledvocabulary.studyAssayPlatform.xevo_qtof_ms_(waters)=Xevo QTof MS (Waters) controlledvocabulary.studyAssayPlatform.xevo_tq_ms_(waters)=Xevo TQ MS (Waters) controlledvocabulary.studyAssayPlatform.xevo_tq-s_(waters)=Xevo TQ-S (Waters) -controlledvocabulary.studyAssayPlatform.other=Other \ No newline at end of file +controlledvocabulary.studyAssayPlatform.other=Other From a68212207166665c6041214fbe4943dba16a5177 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 21 Sep 2020 16:09:56 -0400 Subject: [PATCH 041/113] Modifications for preserving folder structure inside zip archives containing shape files. (#6873) --- .../harvard/iq/dataverse/util/FileUtil.java | 696 +++++++++--------- .../iq/dataverse/util/ShapefileHandler.java | 46 +- 2 files changed, 395 insertions(+), 347 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 67b2e817fca..7ed9970fe13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -720,124 +720,128 @@ public static String generateOriginalExtension(String fileType) { } public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException { - List datafiles = new ArrayList<>(); - - String warningMessage = null; - + List datafiles = new ArrayList<>(); + + String warningMessage = null; + // save the file, in the temporary location for now: - Path tempFile = null; - + Path tempFile = null; + Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId()); - String finalType = null; - if (newStorageIdentifier == null) { - if (getFilesTempDirectory() != null) { - tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); - // "temporary" location is the key here; this is why we are not using - // the DataStore framework for this - the assumption is that - // temp files will always be stored on the local filesystem. - // -- L.A. Jul. 2014 - logger.fine("Will attempt to save the file as: " + tempFile.toString()); - Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); - - // A file size check, before we do anything else: - // (note that "no size limit set" = "unlimited") - // (also note, that if this is a zip file, we'll be checking - // the size limit for each of the individual unpacked files) - Long fileSize = tempFile.toFile().length(); - if (fileSizeLimit != null && fileSize > fileSizeLimit) { - try {tempFile.toFile().delete();} catch (Exception ex) {} - throw new IOException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); - } - - } else { - throw new IOException("Temp directory is not configured."); - } - logger.fine("mime type supplied: " + suppliedContentType); - // Let's try our own utilities (Jhove, etc.) to determine the file type - // of the uploaded file. (We may already have a mime type supplied for this - // file - maybe the type that the browser recognized on upload; or, if - // it's a harvest, maybe the remote server has already given us the type - // for this file... with our own type utility we may or may not do better - // than the type supplied: - // -- L.A. - String recognizedType = null; - - try { - recognizedType = determineFileType(tempFile.toFile(), fileName); - logger.fine("File utility recognized the file as " + recognizedType); - if (recognizedType != null && !recognizedType.equals("")) { - if(useRecognizedType(suppliedContentType, recognizedType)) { - finalType=recognizedType; - } - } - - } catch (Exception ex) { - logger.warning("Failed to run the file utility mime type check on file " + fileName); - } - - if (finalType == null) { - finalType = (suppliedContentType == null || suppliedContentType.equals("")) - ? MIME_TYPE_UNDETERMINED_DEFAULT - : suppliedContentType; - } - - // A few special cases: - - // if this is a gzipped FITS file, we'll uncompress it, and ingest it as - // a regular FITS file: - - if (finalType.equals("application/fits-gzipped")) { - - InputStream uncompressedIn = null; - String finalFileName = fileName; - // if the file name had the ".gz" extension, remove it, - // since we are going to uncompress it: - if (fileName != null && fileName.matches(".*\\.gz$")) { - finalFileName = fileName.replaceAll("\\.gz$", ""); - } - - DataFile datafile = null; - try { - uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile())); - File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit); - datafile = createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, systemConfig.getFileFixityChecksumAlgorithm()); - } catch (IOException | FileExceedsMaxSizeException ioex) { - datafile = null; - } finally { - if (uncompressedIn != null) { - try {uncompressedIn.close();} catch (IOException e) {} - } - } - - // If we were able to produce an uncompressed file, we'll use it - // to create and return a final DataFile; if not, we're not going - // to do anything - and then a new DataFile will be created further - // down, from the original, uncompressed file. - if (datafile != null) { - // remove the compressed temp file: - try { - tempFile.toFile().delete(); - } catch (SecurityException ex) { - // (this is very non-fatal) - logger.warning("Failed to delete temporary file " + tempFile.toString()); - } - - datafiles.add(datafile); - return datafiles; - } - - // If it's a ZIP file, we are going to unpack it and create multiple - // DataFile objects from its contents: - } else if (finalType.equals("application/zip")) { - - ZipInputStream unZippedIn = null; - ZipEntry zipEntry = null; - - int fileNumberLimit = systemConfig.getZipUploadFilesLimit(); - - try { - Charset charset = null; - /* + String finalType = null; + if (newStorageIdentifier == null) { + if (getFilesTempDirectory() != null) { + tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); + // "temporary" location is the key here; this is why we are not using + // the DataStore framework for this - the assumption is that + // temp files will always be stored on the local filesystem. + // -- L.A. Jul. 2014 + logger.fine("Will attempt to save the file as: " + tempFile.toString()); + Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); + + // A file size check, before we do anything else: + // (note that "no size limit set" = "unlimited") + // (also note, that if this is a zip file, we'll be checking + // the size limit for each of the individual unpacked files) + Long fileSize = tempFile.toFile().length(); + if (fileSizeLimit != null && fileSize > fileSizeLimit) { + try { + tempFile.toFile().delete(); + } catch (Exception ex) { + } + throw new IOException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); + } + + } else { + throw new IOException("Temp directory is not configured."); + } + logger.fine("mime type supplied: " + suppliedContentType); + // Let's try our own utilities (Jhove, etc.) to determine the file type + // of the uploaded file. (We may already have a mime type supplied for this + // file - maybe the type that the browser recognized on upload; or, if + // it's a harvest, maybe the remote server has already given us the type + // for this file... with our own type utility we may or may not do better + // than the type supplied: + // -- L.A. + String recognizedType = null; + + try { + recognizedType = determineFileType(tempFile.toFile(), fileName); + logger.fine("File utility recognized the file as " + recognizedType); + if (recognizedType != null && !recognizedType.equals("")) { + if (useRecognizedType(suppliedContentType, recognizedType)) { + finalType = recognizedType; + } + } + + } catch (Exception ex) { + logger.warning("Failed to run the file utility mime type check on file " + fileName); + } + + if (finalType == null) { + finalType = (suppliedContentType == null || suppliedContentType.equals("")) + ? MIME_TYPE_UNDETERMINED_DEFAULT + : suppliedContentType; + } + + // A few special cases: + // if this is a gzipped FITS file, we'll uncompress it, and ingest it as + // a regular FITS file: + if (finalType.equals("application/fits-gzipped")) { + + InputStream uncompressedIn = null; + String finalFileName = fileName; + // if the file name had the ".gz" extension, remove it, + // since we are going to uncompress it: + if (fileName != null && fileName.matches(".*\\.gz$")) { + finalFileName = fileName.replaceAll("\\.gz$", ""); + } + + DataFile datafile = null; + try { + uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile())); + File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit); + datafile = createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, systemConfig.getFileFixityChecksumAlgorithm()); + } catch (IOException | FileExceedsMaxSizeException ioex) { + datafile = null; + } finally { + if (uncompressedIn != null) { + try { + uncompressedIn.close(); + } catch (IOException e) { + } + } + } + + // If we were able to produce an uncompressed file, we'll use it + // to create and return a final DataFile; if not, we're not going + // to do anything - and then a new DataFile will be created further + // down, from the original, uncompressed file. + if (datafile != null) { + // remove the compressed temp file: + try { + tempFile.toFile().delete(); + } catch (SecurityException ex) { + // (this is very non-fatal) + logger.warning("Failed to delete temporary file " + tempFile.toString()); + } + + datafiles.add(datafile); + return datafiles; + } + + // If it's a ZIP file, we are going to unpack it and create multiple + // DataFile objects from its contents: + } else if (finalType.equals("application/zip")) { + + ZipInputStream unZippedIn = null; + ZipEntry zipEntry = null; + + int fileNumberLimit = systemConfig.getZipUploadFilesLimit(); + + try { + Charset charset = null; + /* TODO: (?) We may want to investigate somehow letting the user specify the charset for the filenames in the zip file... @@ -857,126 +861,129 @@ public static List createDataFiles(DatasetVersion version, InputStream } } - */ - - if (charset != null) { - unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset); - } else { - unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile())); - } - - while (true) { - try { - zipEntry = unZippedIn.getNextEntry(); - } catch (IllegalArgumentException iaex) { - // Note: - // ZipInputStream documentation doesn't even mention that - // getNextEntry() throws an IllegalArgumentException! - // but that's what happens if the file name of the next - // entry is not valid in the current CharSet. - // -- L.A. - warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is."; - logger.warning(warningMessage); - throw new IOException(); - } - - if (zipEntry == null) { - break; - } - // Note that some zip entries may be directories - we - // simply skip them: - - if (!zipEntry.isDirectory()) { - if (datafiles.size() > fileNumberLimit) { - logger.warning("Zip upload - too many files."); - warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + - "); please upload a zip archive with fewer files, if you want them to be ingested " + - "as individual DataFiles."; - throw new IOException(); - } - - String fileEntryName = zipEntry.getName(); - logger.fine("ZipEntry, file: " + fileEntryName); - - if (fileEntryName != null && !fileEntryName.equals("")) { - - String shortName = fileEntryName.replaceFirst("^.*[\\/]", ""); - - // Check if it's a "fake" file - a zip archive entry - // created for a MacOS X filesystem element: (these - // start with "._") - if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { - // OK, this seems like an OK file entry - we'll try - // to read it and create a DataFile with it: - - File unZippedTempFile = saveInputStreamInTempFile(unZippedIn, fileSizeLimit); - DataFile datafile = createSingleDataFile(version, unZippedTempFile, null, shortName, - MIME_TYPE_UNDETERMINED_DEFAULT, - systemConfig.getFileFixityChecksumAlgorithm(), null, false); - - if (!fileEntryName.equals(shortName)) { - // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes), - // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all - // the leading, trailing and duplicate slashes; then replace all the characters that - // don't pass our validation rules. - String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", ""); - directoryName = StringUtil.sanitizeFileDirectory(directoryName, true); - // if (!"".equals(directoryName)) { - if (!StringUtil.isEmpty(directoryName)) { - logger.fine("setting the directory label to " + directoryName); - datafile.getFileMetadata().setDirectoryLabel(directoryName); - } - } - - if (datafile != null) { - // We have created this datafile with the mime type "unknown"; - // Now that we have it saved in a temporary location, - // let's try and determine its real type: - - String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(); - - try { - recognizedType = determineFileType(new File(tempFileName), shortName); - logger.fine("File utility recognized unzipped file as " + recognizedType); - if (recognizedType != null && !recognizedType.equals("")) { - datafile.setContentType(recognizedType); - } - } catch (Exception ex) { - logger.warning("Failed to run the file utility mime type check on file " + fileName); - } - - datafiles.add(datafile); - } - } - } - } - unZippedIn.closeEntry(); - - } - - } catch (IOException ioex) { - // just clear the datafiles list and let - // ingest default to creating a single DataFile out - // of the unzipped file. - logger.warning("Unzipping failed; rolling back to saving the file as is."); - if (warningMessage == null) { - warningMessage = "Failed to unzip the file. Saving the file as is."; - } - - datafiles.clear(); - } catch (FileExceedsMaxSizeException femsx) { - logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage()); - warningMessage = femsx.getMessage() + "; saving the zip file as is, unzipped."; - datafiles.clear(); - } finally { - if (unZippedIn != null) { - try {unZippedIn.close();} catch (Exception zEx) {} - } - } - if (datafiles.size() > 0) { - // link the data files to the dataset/version: - // (except we no longer want to do this! -- 4.6) - /*Iterator itf = datafiles.iterator(); + */ + + if (charset != null) { + unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset); + } else { + unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile())); + } + + while (true) { + try { + zipEntry = unZippedIn.getNextEntry(); + } catch (IllegalArgumentException iaex) { + // Note: + // ZipInputStream documentation doesn't even mention that + // getNextEntry() throws an IllegalArgumentException! + // but that's what happens if the file name of the next + // entry is not valid in the current CharSet. + // -- L.A. + warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is."; + logger.warning(warningMessage); + throw new IOException(); + } + + if (zipEntry == null) { + break; + } + // Note that some zip entries may be directories - we + // simply skip them: + + if (!zipEntry.isDirectory()) { + if (datafiles.size() > fileNumberLimit) { + logger.warning("Zip upload - too many files."); + warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + + "); please upload a zip archive with fewer files, if you want them to be ingested " + + "as individual DataFiles."; + throw new IOException(); + } + + String fileEntryName = zipEntry.getName(); + logger.fine("ZipEntry, file: " + fileEntryName); + + if (fileEntryName != null && !fileEntryName.equals("")) { + + String shortName = fileEntryName.replaceFirst("^.*[\\/]", ""); + + // Check if it's a "fake" file - a zip archive entry + // created for a MacOS X filesystem element: (these + // start with "._") + if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { + // OK, this seems like an OK file entry - we'll try + // to read it and create a DataFile with it: + + File unZippedTempFile = saveInputStreamInTempFile(unZippedIn, fileSizeLimit); + DataFile datafile = createSingleDataFile(version, unZippedTempFile, null, shortName, + MIME_TYPE_UNDETERMINED_DEFAULT, + systemConfig.getFileFixityChecksumAlgorithm(), null, false); + + if (!fileEntryName.equals(shortName)) { + // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes), + // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all + // the leading, trailing and duplicate slashes; then replace all the characters that + // don't pass our validation rules. + String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", ""); + directoryName = StringUtil.sanitizeFileDirectory(directoryName, true); + // if (!"".equals(directoryName)) { + if (!StringUtil.isEmpty(directoryName)) { + logger.fine("setting the directory label to " + directoryName); + datafile.getFileMetadata().setDirectoryLabel(directoryName); + } + } + + if (datafile != null) { + // We have created this datafile with the mime type "unknown"; + // Now that we have it saved in a temporary location, + // let's try and determine its real type: + + String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(); + + try { + recognizedType = determineFileType(new File(tempFileName), shortName); + logger.fine("File utility recognized unzipped file as " + recognizedType); + if (recognizedType != null && !recognizedType.equals("")) { + datafile.setContentType(recognizedType); + } + } catch (Exception ex) { + logger.warning("Failed to run the file utility mime type check on file " + fileName); + } + + datafiles.add(datafile); + } + } + } + } + unZippedIn.closeEntry(); + + } + + } catch (IOException ioex) { + // just clear the datafiles list and let + // ingest default to creating a single DataFile out + // of the unzipped file. + logger.warning("Unzipping failed; rolling back to saving the file as is."); + if (warningMessage == null) { + warningMessage = "Failed to unzip the file. Saving the file as is."; + } + + datafiles.clear(); + } catch (FileExceedsMaxSizeException femsx) { + logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage()); + warningMessage = femsx.getMessage() + "; saving the zip file as is, unzipped."; + datafiles.clear(); + } finally { + if (unZippedIn != null) { + try { + unZippedIn.close(); + } catch (Exception zEx) { + } + } + } + if (datafiles.size() > 0) { + // link the data files to the dataset/version: + // (except we no longer want to do this! -- 4.6) + /*Iterator itf = datafiles.iterator(); while (itf.hasNext()) { DataFile datafile = itf.next(); datafile.setOwner(version.getDataset()); @@ -988,125 +995,138 @@ public static List createDataFiles(DatasetVersion version, InputStream version.getDataset().getFiles().add(datafile); } */ - // remove the uploaded zip file: - try { - Files.delete(tempFile); - } catch (IOException ioex) { - // do nothing - it's just a temp file. - logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); - } - // and return: - return datafiles; - } - - } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) { - // Shape files may have to be split into multiple files, - // one zip archive per each complete set of shape files: - - // File rezipFolder = new File(this.getFilesTempDirectory()); - File rezipFolder = getShapefileUnzipTempDirectory(); - - IngestServiceShapefileHelper shpIngestHelper; - shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder); - - boolean didProcessWork = shpIngestHelper.processFile(); - if (!(didProcessWork)) { - logger.severe("Processing of zipped shapefile failed."); - return null; - } - - try { - for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) { - FileInputStream finalFileInputStream = new FileInputStream(finalFile); - finalType = determineContentType(finalFile); - if (finalType == null) { - logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'"); - continue; - } - - File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit); - DataFile new_datafile = createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, systemConfig.getFileFixityChecksumAlgorithm()); - if (new_datafile != null) { - datafiles.add(new_datafile); - } else { - logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName()); - } - finalFileInputStream.close(); - - } - } catch (FileExceedsMaxSizeException femsx) { - logger.severe("One of the unzipped shape files exceeded the size limit; giving up. " + femsx.getMessage()); - datafiles.clear(); - } - - // Delete the temp directory used for unzipping - // The try-catch is due to error encountered in using NFS for stocking file, - // cf. https://github.com/IQSS/dataverse/issues/5909 - try { - FileUtils.deleteDirectory(rezipFolder); - } catch (IOException ioex) { - // do nothing - it's a tempo folder. - logger.warning("Could not remove temp folder, error message : " + ioex.getMessage()); - } - - if (datafiles.size() > 0) { - // remove the uploaded zip file: - try { - Files.delete(tempFile); - } catch (IOException ioex) { - // do nothing - it's just a temp file. - logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); - } catch (SecurityException se) { - logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: " - + se.getMessage()); - } - return datafiles; - } else { - logger.severe("No files added from directory of rezipped shapefiles"); - } - return null; - - } - } else { - // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied - finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; - String type = determineFileTypeByExtension(fileName); - if (!StringUtils.isBlank(type)) { - //Use rules for deciding when to trust browser supplied type - if (useRecognizedType(finalType, type)) { - finalType = type; - } - logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType); - } - } + // remove the uploaded zip file: + try { + Files.delete(tempFile); + } catch (IOException ioex) { + // do nothing - it's just a temp file. + logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); + } + // and return: + return datafiles; + } + + } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) { + // Shape files may have to be split into multiple files, + // one zip archive per each complete set of shape files: + + // File rezipFolder = new File(this.getFilesTempDirectory()); + File rezipFolder = getShapefileUnzipTempDirectory(); + + IngestServiceShapefileHelper shpIngestHelper; + shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder); + + boolean didProcessWork = shpIngestHelper.processFile(); + if (!(didProcessWork)) { + logger.severe("Processing of zipped shapefile failed."); + return null; + } + + try { + for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) { + FileInputStream finalFileInputStream = new FileInputStream(finalFile); + finalType = determineContentType(finalFile); + if (finalType == null) { + logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'"); + continue; + } + + File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit); + DataFile new_datafile = createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, systemConfig.getFileFixityChecksumAlgorithm()); + String directoryName = null; + String absolutePathName = finalFile.getParent(); + if (absolutePathName != null) { + if (absolutePathName.length() > rezipFolder.toString().length()) { + // This file lives in a subfolder - we want to + // preserve it in the FileMetadata: + directoryName = absolutePathName.substring(rezipFolder.toString().length() + 1); + + if (!StringUtil.isEmpty(directoryName)) { + new_datafile.getFileMetadata().setDirectoryLabel(directoryName); + } + } + } + if (new_datafile != null) { + datafiles.add(new_datafile); + } else { + logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName()); + } + finalFileInputStream.close(); + + } + } catch (FileExceedsMaxSizeException femsx) { + logger.severe("One of the unzipped shape files exceeded the size limit; giving up. " + femsx.getMessage()); + datafiles.clear(); + } + + // Delete the temp directory used for unzipping + // The try-catch is due to error encountered in using NFS for stocking file, + // cf. https://github.com/IQSS/dataverse/issues/5909 + try { + FileUtils.deleteDirectory(rezipFolder); + } catch (IOException ioex) { + // do nothing - it's a tempo folder. + logger.warning("Could not remove temp folder, error message : " + ioex.getMessage()); + } + + if (datafiles.size() > 0) { + // remove the uploaded zip file: + try { + Files.delete(tempFile); + } catch (IOException ioex) { + // do nothing - it's just a temp file. + logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); + } catch (SecurityException se) { + logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: " + + se.getMessage()); + } + return datafiles; + } else { + logger.severe("No files added from directory of rezipped shapefiles"); + } + return null; + + } + } else { + // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied + finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; + String type = determineFileTypeByExtension(fileName); + if (!StringUtils.isBlank(type)) { + //Use rules for deciding when to trust browser supplied type + if (useRecognizedType(finalType, type)) { + finalType = type; + } + logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType); + } + } // Finally, if none of the special cases above were applicable (or // if we were unable to unpack an uploaded file, etc.), we'll just // create and return a single DataFile: File newFile = null; - if(tempFile!=null) { - newFile = tempFile.toFile(); + if (tempFile != null) { + newFile = tempFile.toFile(); } ChecksumType checkSumType = DataFile.ChecksumType.MD5; - if(newStorageIdentifier==null) { - checkSumType=systemConfig.getFileFixityChecksumAlgorithm(); + if (newStorageIdentifier == null) { + checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); } - + DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum); File f = null; - if(tempFile!=null) { - f=tempFile.toFile(); + if (tempFile != null) { + f = tempFile.toFile(); } - if (datafile != null && ((f != null) || (newStorageIdentifier!=null))) { - + if (datafile != null && ((f != null) || (newStorageIdentifier != null))) { + if (warningMessage != null) { createIngestFailureReport(datafile, warningMessage); datafile.SetIngestProblem(); } datafiles.add(datafile); - + return datafiles; } - + return null; } // end createDataFiles diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index e147f5fed0b..c4ac55a5207 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -273,6 +273,13 @@ private String getFileBasename(String fileName){ } return unzipFileName; } + + private String getFolderName(String fileName){ + if (fileName==null){ + return null; + } + return new File(fileName).getParent(); + } /* Unzip the files to the directory, FLATTENING the directory structure @@ -280,7 +287,7 @@ private String getFileBasename(String fileName){ */ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File target_directory){ - //logger.info("unzipFilesToDirectory: " + target_directory.getAbsolutePath() ); + logger.fine("unzipFilesToDirectory: " + target_directory.getAbsolutePath() ); if (zipfile_input_stream== null){ this.addErrorMessage("unzipFilesToDirectory. The zipfile_input_stream is null."); @@ -301,7 +308,7 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File while((origEntry = zipStream.getNextEntry())!=null){ String zentryFileName = origEntry.getName(); - //logger.info("\nOriginal entry name: " + origEntry); + logger.fine("\nOriginal entry name: " + origEntry); if (this.isFileToSkip(zentryFileName)){ logger.fine("Skip file"); @@ -312,20 +319,28 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File if (origEntry.isDirectory()) { //logger.info("Subdirectory found!"); logger.fine("Skip directory"); - //String dirpath = target_directory.getAbsolutePath() + "/" + zentryFileName; - //createDirectory(dirpath); + String dirpath = target_directory.getAbsolutePath() + "/" + zentryFileName; + createDirectory(dirpath); continue; // Continue to next Entry } logger.fine("file found!"); // Write the file String unzipFileName = this.getFileBasename(zentryFileName); + String unzipFolderName = this.getFolderName(zentryFileName); + + String unzipFilePath = unzipFileName; + if (unzipFolderName != null) { + unzipFilePath = unzipFolderName + "/" + unzipFileName; + } + if (unzipFileName==null){ logger.warning("Zip Entry Basename is an empty string: " + zentryFileName); continue; } - String outpath = target_directory.getAbsolutePath() + "/" + unzipFileName; + //String outpath = target_directory.getAbsolutePath() + "/" + unzipFileName; + String outpath = target_directory.getAbsolutePath() + "/" + unzipFilePath; if (unzippedFileNames.contains(outpath)){ logger.info("Potential name collision. Avoiding duplicate files in 'collapsed' zip directories. Skipping file: " + zentryFileName); continue; @@ -493,6 +508,8 @@ private boolean redistributeFilesFromZip(String source_dirname, String target_di //this.msg("source_dirname: "+ source_dirname); //msgt("create zipped shapefile"); + // Make sure the parent folder(s) are there: + createDirectory(new File(target_zipfile_name).getParentFile()); ZipMaker zip_maker = new ZipMaker(namesToZip, source_dirname, target_zipfile_name); this.addFinalRezippedFile(target_zipfile_name); @@ -526,6 +543,11 @@ private boolean straightFileCopy(String sourceFileName, String targetFileName){ File source_file = new File(sourceFileName); File target_file = new File(targetFileName); + + if (target_file.getParentFile() != null) { + // Make sure the parent folder(s) are there: + createDirectory(target_file.getParentFile()); + } try { Files.copy(source_file.toPath(), target_file.toPath(), REPLACE_EXISTING); } catch (IOException ex) { @@ -681,22 +703,28 @@ private boolean examineZipfile(FileInputStream zip_file_stream){ //createDirectory(dirpath); continue; } - + String unzipFileName = this.getFileBasename(zentryFileName); if (unzipFileName==null){ logger.warning("Zip Entry Basename is an empty string: " + zentryFileName); continue; } + String unzipFolderName = this.getFolderName(zentryFileName); + + String unzipFilePath = unzipFileName; + if (unzipFolderName != null) { + unzipFilePath = unzipFolderName + "/" + unzipFileName; + } String s = String.format("Entry: %s len %d added %TD", - unzipFileName, entry.getSize(), + unzipFilePath, entry.getSize(), new Date(entry.getTime())); if (!this.filesListInDir.contains(s)){ this.filesListInDir.add(s); - updateFileGroupHash(unzipFileName); - this.filesizeHash.put(unzipFileName, entry.getSize()); + updateFileGroupHash(unzipFilePath); + this.filesizeHash.put(unzipFilePath, entry.getSize()); } } // end while From 1b24266d6520c9d0faafb736f343ef5358f5c3dc Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 21 Sep 2020 16:24:11 -0400 Subject: [PATCH 042/113] edited a comment in the shape file handler utility, to indicate the changes that have been made. (#6873) --- .../harvard/iq/dataverse/util/ShapefileHandler.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index c4ac55a5207..d5e40dffaf3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -281,10 +281,14 @@ private String getFolderName(String fileName){ return new File(fileName).getParent(); } /* - Unzip the files to the directory, FLATTENING the directory structure - - Any colliding names will result in overwrites - + We used to unzip the files to the directory, FLATTENING the directory structure + Any colliding names would result in overwrites + HOWEVER, starting with v5.1, we are now preserving the folder structure + inside the uploaded zip file (issue #6873). To achieve this, we recreate + all the folders as they appear in the original zip archive, and as we + rezip any found shape file sets. The FileUtil then preserve any such + subfolders in the FileMetadata of the newly created DataFiles. + (-- L.A. 09/2020) */ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File target_directory){ logger.fine("unzipFilesToDirectory: " + target_directory.getAbsolutePath() ); From 41985bb26ffa0921d030386923eb9fb96ef24c1d Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 21 Sep 2020 22:51:09 -0400 Subject: [PATCH 043/113] Added a RestAssured test for shape file upload and processing. It's very simple, but it appears that we weren't testing it at all. (#6873) --- scripts/search/data/shape/shapefile.zip | Bin 0 -> 57541 bytes .../edu/harvard/iq/dataverse/api/FilesIT.java | 62 ++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 scripts/search/data/shape/shapefile.zip diff --git a/scripts/search/data/shape/shapefile.zip b/scripts/search/data/shape/shapefile.zip new file mode 100644 index 0000000000000000000000000000000000000000..0bf5ee98c6cd0d1f3b1d08d68dbc307749662c16 GIT binary patch literal 57541 zcmb5U1yJSOwk^DIcN%Y88+UhS6WUe{K7&B1>LO{X;{#n+3vn%~;=f5`a00MxMi=nBVwXumKqnaul0K!qj zLhtHxb#X@kfI;4a0RWJgpI0k>+W4o11w_R9%7A|tPbm33WdQ*I(ET?XhIUTQcD8!< zE{4_?MtV+0=5}`0PI@d%%q)7I%+?I{j+Xx#1>JW|-4*Kp8YLz{&n}Y*v%fQP&n&~> z<^zo)T#?cvGxO02COA_){t*^{lPT>)HVnS?BrF*#|L%1QT`^@X|lv^{A97y{p^74aY#FIaK zhv~j2PY!jd;fu7`A0oc!Cjzf2^OpgAKe?k_TryZOmCQtf6HVMhGXuYK%h zg;uE_J(Oj-uwPf6vDI?Eo>&%fw>kQpbeu{~|8&Lpg_})|O%-CKQWdwrN^y&y$vmmi zQyl7AAuuC>Tp0LHMw0WcB!$>3ye|36IzpG7~_F%2S^OS;#dEeG@(=~{f+C{Gm?9#H z5!2hm1TlVN=45lD(YTx?6HCtpyK6FuvY|M_QFSer9G~!oI2ihGD>rLilV%f3PFd-K z`(sBg9gSX{4>ukg7mHn8DR!6BlbaUV5J-s7_MJ1P)|;t0SVEB-H)DGP$nk6I70+?1 zup1n_V%QfB{&9LAjgxx4mmklbm(O0(t!1dEHNWsEo^F>FroMlz8}_`F2V4hW&v>(z zB8jzPz6-aTGJ$o5{hVoMu?(8>E*yx^k%g;Rs-acd6pjfqy}K=HRHpT(yNPxRK%Ns2 z=iO5Al5hwVnylQ}>ZdJ3k^sWy4!O}&PBof`td2%4qt<-|O2m{+kX%-`Q1|i0ESq^Q zbV>gDGL~320tX3I@x5eF86&URoytY>7UOYPa6GIt3r`rInG;0!oC_)Oa*nR?rx|v- zMqWovXH|If*x^!9Oc6iw{^*H*xd9R!UzxKsvoqf`y!hFmY>4dH&dWTQ61j6m?N%4S zuu$bVP{ZyWyWpg0<1iH0GB3m^Ak6MC)IB8X^pP?L&`jpk5!e@UpxeklO|IQL!E%M|m9k}B(cIFfrOL(!D(B}0c|7esvWh#L^5Q%j*_D-YeT1LZkxHhl>#XCDS?|%$Ttlc z*w`h1$)k){G~8+Fh|geOzT>9A>W+hpX5(A0)aU0%9(OVYbXt=I7{u*No*Ca+1Yz0V z8m9UQ1bhc=gL;xP=ry(V3S6DI+$M`4`ra#2PNyn78gwS`z(W(uDMp~?`aGY9bQLyu z^7fEM$JqqOOr-L)F-Pxx_%c3f`F_AK=v7$0f)*E4oWz_eGX;IWvjnu12JCOtHgo0E zRJ@GH7Gp?p$|P>s7u{22H1E9@-$FnVUN~{Ndt%jBIdh{OD$nXGGS3&;idu^@-pPbQM&O}q-g_v??#c1lwJl39e z$`4-dn{z0o85^^)%+vU-+JW20`Y~ zL)a*y*O!@x;ro7BbD`Ya_u#Eu&s=8XHv-q}DE(7G7AIt+rl5++JLo+Hy`j|{-Wc4~ z3NkVEi7Jzm#7~%UD49+eJy{9NKwR)+og#2`FUxgte8KL#%5uc3Pd6vg`1W3o4{q*u zYW~N7e?MvgTX#(GMisy)|0N}9tcY?t23I9vNmW$e&B+Ythr=#dm**LTr$+JP#$_P` z?L8ETn} z5j#-hTLd@rmKG1IRuh<>nl>;f3r2I zmldTgfEcMjx9E*~1rrScluFx@+SR1dlX5bTh(%5%1W_+M!_{Q>Xx`o**KBvgb3HCb z?;U#2(ue_bJ$(t3DP}j*E!dYxwj;1(>umb(3u<#%hqyOZb)JcruL07KcuDD&_$iBM z=1va{P>pTGipK=hACV>}B6asb1-e)#7n@fe@@HnP7gDxGUX|ZJlOd7WGA}(rbEGal zx3_Kxgb&({Mg>NhM7)Doh4s=0W&L(d{4HAFWb|>-L1qlV+t*I>x;Q>yf8JB2N`;F3 z6L$&iZDVgmisAIBD(Ce`7`#F6j?xo``bU4hMFU7m+z!g#_>hpO>}x&9`~j*txnEnN zL(sE=H)&ivlCsmi)C9Kif{3ozYP3;FD|Er`quCX;QK0kC2z)on74NQTGu^0(`Aa#e_^|dlbc>fn>mNGxkcN{P6N_jvH>{0>VL6Qjio#?U z?a@~`Y2qE@<3jLz!uUW_A7J~Gm^g7!Nm)TmwqKbi9g>aXE#g(j8D#W+z3M>i8*p)c zrcwE7R297E&RdsuB@D3+y!Y>Kxe&-#JHpf7CT2U9b*n$QyAMJ{)V->xsMrFQ@v=Rt z5)mI|VSCMZzUuTI)2S$oo8@ouE@wL?Tenfw*P9F8CuzP$tvrOgF=&c%m^S2`Eady_ zW=#l3o!(DwJPHBcgV>UFD|8nMxjXLXC_lvXdr;cnGD7AYUHEnIjW(NeykJ@?68i1m z9{m10Yb~*|CKdl=t=5qLYog-*KM@t1SsUsg2h4#jvq@7nMhoc#a=3Kz&qfjmOdyv= zpAkuD1BKL)lUTr!NGAl7!$~I?fA))rXwG~#EbpC+mTuXr&ynHW=^%+w1ks1s+k+;|w zb~p&PD0r@PN@wy2$(X=qned~g0_Dtmg|qsgR{J3M24TFvjG#m7;|v(Un;L+9*XMWD z7mVE^_`OA#vrXS}2`pVOfR!*}F*PJiF{Ic00~6)|{lgw6?g~ElQnYm15LLpEwAct4 z%jipkKK8N$e5ySH$`v@#mE0kP5pk#yUvU90!XI?}Y77{A@B@dh4_Am6mr$J{nu3V^ z>TCsYH-8XMs*$`LFwGpkxLrZoUE&IctP9{oD?L7Z2>y3heb;BmI++3hmVD6ucLKxM z(DZ+D)v1nLJi)M}VHqN`Kuy-S*_9|NgQtc(+r|km83N zb6&|QSl*e;IFLBK{aAg7wEh}3**`WAi*ebLso{gBy=dt`BnwEjL8xuG{9cXJTe=BWGO>%sLqrP2Oz zZ!(AIqnesX{`|*X(J>NqF8{{6r4ql*hTom{%lmXeh8oNu+1ty5p3Zc~#6uLu$J_Bk zRZS32l{8os0@6ch4eosB~w7B)tUo?~W z_~F{l^6Xc8EY13n>$z5Lw}0`qwIjXrg!Xo2*8A0*jl~byZ@ru<=6&(|;f36YxWmkQ zT;;RLOQ%l$#pI~B>&(00KZiejFWcTVxdq3}_SD^&Wr_KZko+t|eA5_5P$HG|21^W( z4&`1pwDUXF1ZeIRPCts(I_o}&z8|{T>f-)Py#FwoXI$xg51IG$oDrx!PCfG*C@`Zs z6Ri7Zcd_7r>g;2zT(9ASDoTLg;)DK|@6!7?5l>a8*Ng4P{^5S?dUvQ`v&hHq@AR#U zjWfkmz&FOqDU`{&51-8UqBp(Y$9i^xg-Wx%mwIQyHmrzo$07mAsVfe2`_n8^8M1%IKZN-AAIp`wZc|!{l*xexwuW z{&_E%op#mX*!h6Zpaw0Q{a1RUL6d7 zQ10$sT^eKgVEdLIgy-!0jh^P#NArTskCvcyMexs#k{^c~^rnUQ{ik{5UcE*_cFP|V z@Ne}-?LMdbcVzsXe(;}m{Iv7^7zKw$9R!xM??XQjKD#*A)ji&Icf-c9m|x+fs#g45 zLVUSel;?LMa<dcF~=0xtrJ>-4;&^Wl|&F1r_iGI%<%t{h{uk9JzsDkih_45O-Uwr%cT2%7+;FVJw zl{ZVTih>Y(?ORrMOLxV7e?$c58cjXq?A%d4Z%pC);oZ&B+WwJJSjL9$$?Ma{-r;x5 z8fvsr$b=3%gm!o|3+JjPK|M^Dy2d8V^fO9LiD;mb2;8{0*5Bcq1c5KOTOE~`6#M;J z%(*Wdy@}q=YneaHJLs2-vLYNi2C zhv-vwVW#)O4+W7?yfbSJnv)k%Zd$Z=JCLqO-C{>jd{#CUS|Eh~!hqWY5xqQdV5q31 zz3~r!*pjEJY;ZWw4DW}S!V%zBcm1@F)b8H9=>){SN%zDnQIhu$L>%cwBZPMykR%7Q z{u1av7e}9kJnuHZ6f-S*192&(4fe4ZcoIx0r zZU(BDTnKOR4`I%*t`^=dA+=ze!+7RU2S^A*q8Gsm%*S-v=fRAPrAgbTVe%LC<61-X zi~J#kGK4s;cSz{w;Nn7TyMo(LGd#I;~%{&2({r4PFVWPw#^e6I_8>>*E1R(a5HD)553zq^-_aZ&#&g4L;t-<8XuFpD zuP&@HT%}aFI~!4|Y-?R=_@O31dbkjS5ZYIolyccWt6cD#!51_(ASYwxl3KHE#)E^e@KG;MCvAn zhzLURu$o|V;$$|Zt~E-;adde1f+@B_LMR|Lgum?Lb;+eF>zt8c0Ts(=g(r4+o}%(! zoA@TAu}3=pl4aR@?baav@cIh_(L+3ps7g**2+Gk_ITC;^M$qxS& z4jWUz$@XCOZE%lnxfz#KgzR)$a1Xep#fOBP>R3N9Xu^zyndw-`*t{{z~-OYlpQvQMTz4GN<#>)n8W_XEJ|!4qRKeaZrBwk;KyCI-OBQ(ON`MO zJ>=TdYCRQD9bN6`4E$-+{(iCd1cT< zsYwh<=r~X=20mMmP1t8@hp+g&YYu z6*XiM$8wr&FC3Bi06kK?1cA80KA7^jub7%0XI|!=_KK-&d?JAmg!Q<=aS_CmtZ90X z_zS3o1b^eno7nZjmFm{#maFKJ;>P==G4@aSxrEHVz!VZpq>^rlA`cq5VC(`ql8E*G z2qsrM&_HVaGT2e(`ZDdR6itPsH&R<*3>1@j=r|Hj=};tHMRfVwlvH~n&^B3^+t0m` z@)%zY!8@p1XMGSwy2>LkIvw`SGO8GfA>>FfuotvST|C>h`WCIto56o;6C`+q3~4|F z+eOcOdV~h

    nYz)ae-H?BXqwd~ye4($q@XJ2PPZ4MHx|@f3+~&=;bexL>!r5F0iu zNe&hpHA^rI6bS~>QVv7e6^EYpbttERJcfHt_8k+#;0Oj95M)TF7BNvwacMWnP)7qD zny~d@;54&~Nof5&?WIcsG193|ZR_2*VYX>Vx--R<2$bpcaS|Z(WWZbbzt4{wf7|k~ z-vUbG>@v%eHOS?jyegyy}5CkTLe#zXPxj@J!LQG#xU!v}z#^G1tFv zigrQ86|^JufPI4-lnxB6G(t8q3Ks$*6<_Ylmx=xnv8Vh(_^S+jt>xDaMs?n1#phct z(|wU02%kw!H}39GMZ!;Y^j!-}isr%J z@?NR&faA?`s<^#v(1lGr4l=|d=h$01+-MeNs3p3wnKY%~?FUVvI?(X$)lq}*MTZyU zuOt1RxxWoNjF(%9{7-6Y2huI=e%j_rsQl~DnY7N>Ird#|!M#l+i~+Tr!2K~^NOm3{ zz?_AxU-{KMWs}tSu!EjK$Ow0X0L|(C4b&Q4nzJ|_9 zimat*u?&y~y_%3uQ;SV!-vgpXRuwB?z$DO?$%_x`T}z@tyN|2Uf#*ED2Y~oOM|1|o z@>#zCbJA;WIJ)JlDHn%f69Kc~2;t$h;Th1N!N@J)ra2pF<6;=Y#_oJw>2fHhZE#5C z5>mX&=T1POI>c-^M7dZ7eSf(fk(6Gv7s*w!I5${$_yXz}V-H{529^#J312czpIYF9 zkUF?387cK-UTmER*j*)cOYJ~{Df%W$Pvi?;TtYB-SUYkmdNp-yd`UaThBFQ=``x->PwWSO_SleHOSQo8Om$|t2dhOL81ml8B zXrlO;ie7{Owh;NZS6%E-Xz+Y+r=nC!y3tB9w(F zh_ML*^Zzs+H;buor02T=B(_)*#QKGQ z2Yur4l;|UW3pl3e)Bu!y`O!ai82o2>ZiS>J&iJ{KVp*bcS8guO%x0jDim(Oa-CWCl zh^VB1!4hGb;HZ$`HoZFFcTqQm(hq~ia>?NUDUMq}5=1eUx(;l75uN}}2;LL+OkS1A z#a~}Z>4zb~ywUlQ2co}0SxYmhwV;#aOGUNWFg0QNwD@mkJfejl6qNd4(j2om6hDYr z4q7!lg#F9e(Z}Dcwc$&?;eEdgvV1Pxl4(w6#CkFiuu`N+df})_JfQZ;Cr7zA>ioyg zIu9g1cs9E-YH?Kk-e|i&e3kwNWa}6Gq2eBSf1h?nTVqeAf8OCsi=K7;uu_&!hD*Z4}3k#FvPCgexy$=AaqATRW``{l-nuoS2-P+{o9SP1U zB|;0pWkXUU`ZXi{YDHj}f`WuNM6IX*7q0Db_=XFCertP*1x{*XF9l8tB2uI^487gF zjo@cPvCVZ6z44!cOs_%6%i+aT+uEdFe(*1#zvY9ihDkN1?n2kMMhzF_QhWB##-a!f z=k{*Z>B#$Gk0S4mcxmb|c3=pAt|!D_hC#)a!hr-yaSR*-VtSzPAvTEZ)xxA;NSHkD)*LmV>a_((k6GZF=!~hh;3b6t*Z^C!At;FP^jwG;GYSvH+aQy~e)F~J1vB;lM5Cva8K`;QVWDbjo}582p; zs}QApbzpOyj{?BQFs#yGk|K%%c%%Rp!G;n9hY7bML;kC*qB6Aj4H(Jj{z?G_Chprw zH>%)zS3BZO=;F$yDN|8@UcZKiHs#~h9A_(1@Ill($~W7Uzf>Qugb^prb&o;dKrb?( zAScP0;+=AlI2+)~!h(d9IQEr0p?5>f1rx0{W4e2ZrGk10G4~q1Eb}U%c`V{pe z3y-_%hZn&Q`7#!I{(w-=Cpa%2!|B2gB3$G22R1KUfr4ox2jC%lLb5_5#@yr| zu3H#Pk-mWhvoo?*_s&|pxu*TtDro37$@dZHeN;}A|DUuudgKx^m8BFJxgb-#g9uYc zrjEY}_cDT-7X&oWBIjzs{E8JNuFu2R^;ZOs>2ThTqv49@vj=Nk4tL-_YY|z*ZH`V{ee^BbH!M!q?2|)7KS49f~LgB=Q*{KUwx(EG*dif;?Xu&-T)_1h) zAEP9*Y-3+8QB(?PST7P#g=+@>H)BOoGjd@ zos_iDkpgW%uJgAHA&_PIGW10;Py>q;V{=8P4iP5O$kMR`eEVDXPx|yEf}uub#FUgi zQX^%6A~oM)I#W6T&OOBJGBXp&*jh(lN+zKpz8XpZ_nOp7K@wq(>SQNSgG~h~P5LC+6 zvaAR?Rg;X8t{WR>Ps7Bs4oOukQn>Bqq3RgFB>yDn@(VuhdJ>$D9Dko0Om5I!t*Hov zvvZsJ)!I2sw>FjJDW(i^r5gI>pOn*JkyEdAuQmfdrP1*CwXYIm&&?Ca878nA1#&Z` z2^)r&^JhBY?$_^Oz^i{tO`Hijuus@oa0SaK6Jiva9RY{>T|{jGG06oZK;gPMNGv;y zDdpQtzNwrHkS1c=LuVx=k4Icngd&jekDFat*-Y|RnUY+Rr|~^Q1B`WT-MCJ00iJ|_ zG3kV2@KgSI;c@r!CtazeWh1cuU_z0sj1DxI)L#OTXXj+MFo{c0xdg(f;Z3YpCdr*j zIYgQQrF=D8eO~%kI$%mF{^a!$La#8c-*Yg*^rb2z_Ksrw$-qCP3ce>rpbtIH^r-}k&}Ne;06*Yz5J4geoB$X5N1^py zK)_ssC^veBbY2Q-{}eg{x1RLe4HU~(Jqh^^ZhV>Zx0NIs|8b#BZoLsG)tDiW@A!?b zrxh%*e7e=p9tXrXvdtN*3zV}1Fet6`UmqWHE4{2N$3var=( zMa{_~gdY+9JEAAa{W|tEm^_3{d6nwmHkJ&Sx~f9>w3;Z)5&7Q&A`G;J9iqX%v6+S+ zO}&YvDr28vCkG9+^(}&LudF>M5%E%I=C9y>gmfV1 z8vCIzfwU@!ZavjR`81AI)X=iRC(RB?%p}Fij{cUxX6pMi9JwkhPG5^BV3#4ZZT*$% zO5v46z_=h%<8Cpe!DgUSi{hS0Up%fs+QA4}$}65&%gH6u#wma@8@ixDC7BTQD*7Ce z3S^uHij-({Q#Z|@Z-QQn4Frup>HMIG^tW!`3OFMZ=(T98MCJ$ud&!OE&0(p#^4piI z1iRu}6wQ=0(1@3eu)c8^x*4HX9!t2nV5DcmIY^YKk=D_$%>m2n<;?8mH)j37&)~e#bSapPmrYYem^H53_iK$yf2L)@v##(|+gb!a>^NEeU z+oJCp9hMG$jJ+cR=WUiHhes|qD5Ct5kL+Rw--=y${>bR*bH%c9Q~ zFN+@srfWQaWj=wp7mcTzh9M| zEFZgUxK#=vKI^n%j1Fs{K+d2*?xt|PGd~JeEuR+Gnzu;e?ZkB*N~BN)mc5dY}?gw{fjl zcV+)XGyUpYk|j_gFa_YT|Bife@E_cV*#zMrS-Eih{97~d22PYX@J%{P(b$he-Ge5{ zKg(Hnr0Kfc%*M&{{h;{mm#5dg=v_hc>Y8^+JC>X@$Vq$0ciWs#ulVytvvg=ZwA1MX z+gWNKslo@JW}L^PT-N4uuQcSAW3Sv7oVmtScx~r;F|uBpwPW(Wqd2rB6-K0xa_H8t zRL*&H62cY{!4ASq+=l&4zgQxoeQbp)RsN8lUK%d(r4yb#RG=?0Eq56Pd zwd<2hF9eoNwv=TBLU6_Qx5_mR0m?4w&n+J3tg$u2>bg?c2lg;g*f&W721Qb#Vbv6s zD68`jbY`W};yZ*+?gS|2hkt%t05KiJuHpfwN8@Yy>_s>>vBA{n^*M@=nCUkWh$JWG$WBapjWQCPfNF=M|>lF zC$k4{dS|X$Z!B{lCyz&U6v&!?N01KJo18k&qzKwH(w`Z)lo&cqMiZpweD-Z;0suVm zE?AV55GAQd0Swm|MmL)ZDkf zR?dH`rv|CS1CP6yukE>uJuVTh>x+MmrCd)1td*)2?;&F~cKJ^Z{DF_Xbdk|twlben zIzfflvfpB6>^A|%(+o5NoWg})xHjad#qH(3kzvIKkoRbs0X;+Piz3mxtyJ(F^DQgU z_g)L~5f*p9Cf<4@&^42b4vA0?L2h8IIFfpHtG6wqLkt^e(Udro!|FtNfl=KRr5d6u z_2n%M%>^O^lO?z!qE%ZE?>p&Fu?NKj2V;+-4J)e-@9pT*VUey}NY{2UZe*nX(7X(L zcR@Ddr^NXh%~O_0k)2PMhUP-}fh!|v?hrE*k1nt;gawPFF=T{HgX&2`+F9*~wA6+f z^IoLkK{NgtH;%Py=VdgVY}n)bb!F?;nd3-|;*V~PlK2>m6qRk5Kd;tWutUYxzBz&t zJuc&QW^@3`!;|Z8C$&D4zo@o@O^hD|`0I1;l#}?{lg!SD|BjfRzEwXd9MQ@2dlJX{ z5ED1{HNNe+&StY~{dANe^fBU3%Kb}Jeo3ApZydu;$VLajUJT*88^(D^ zq6yxu9Fzy&+m(Fi?+S^Ed5RbApJfcZ!*Q|9ug9)QuK=1EADxCXT`Z%rej2Mey{IwM5i?0xx@1a);!4?B! zjeryVuO+(ZQWSMMWe{uNFm@np#yZ*bzStUyOSd@M%GjL~QcA?2VWt$6$an`p$ZbC6 z55z4*7u0t3SB|X--tp2ywm)C_B60gKr~b_5NGzob69$5!o_5maGlU;+cW^_}l>S=U z6Z8vN@EF9#7t&UCTmO;oZJE@HF>+SAZJc;Net-r01Ituw(@dWBv`jrQoj@lK!xxJ17nH946bVmd^)|No( zh9f7lxvgs@!sNHQYQk&pqC!~Ad;4)YGZvVUM?wI1nuIVajZ3+kwfQM_QNQ_!>yBrF znmgrK&fcSj%IAmq%>2&3+VE=vVxbC`8G>&&&t2L4fHgB>77*IEil5M7F{rQb48=m) zC;Qbh&Xm!7>Y5b8yxZTHbLPLphyoNwHz)@O&Ajy@06ARtmPJJpQ=oHx0Cu?77JN4T zy=oZqzLS3}j0%Fq@u0T7zF6Xy0AourJguCf@%U8qLo5Oavs)w7Rh-x^`KcUxvY9D- zkptuF*Jy)7BXc7)a>!JBM#|)>C-}bJ4k}l~CxCSTA{ND%vUMcPHXwedx-Dc+JbLGZ zEyb~C13m$E0M|s6{Ln+n&cd>rR&pqOt$U@9TQcaO$*Vp}5{=ajbuL-_KlT4ZjRN{*U^ImxU-ItiE>k`v_;&VxSVsw`Nu z5p2EPU!E!wI-dVrPvm+q-aq&;AUkd7fejz2^O_Upblhjw*TP4BMqx93NIpOH=q@hye`++fG<3Lpklm!iPE)amvEH}pnCjRy>r=&S21 znn!ITVFo-#gHdRy+6-mpz%UoGrF7bdZA6K>`!gp*eMIylgyk=ZOg{6fbB$tOe8p5& z-B+)^eg@d(9fCW;(lePy>dP=p8a%ad1^+azmVp&7Z5=jvTHy)iluFucXKvE?ydP zqikE{C5UrntIPea?|`16XPqWE|>N{vz6wxSJ_C0b?veC} zQGp?Q3-95Jq|w>^Z)Yr!2nXZd{U;2F_-O6PlqeG$|8S|~^C5k7bsTcKX`c6TfH`0k z5O^+29Rzx;A^NfPHhpQcX^}_AwIZ5qI9oDdyz6-R7Z~WUP2xF{ z8F3}Qd-!Zl5Hd+Qbrb7h1^pJ&a=uv+K~}W+L_G`eUD%kN&FqqBe3oFAn8>8KxDe^+ zcByoLkGt~6$has6-6%u}95sdq!x>x9Ah{la0lV+ zot;7>(mb1}aH@#z;vpFfl?l;tG==4~1d=AzoU0Ze#9+OMflw#Wc)>s_EMd}B3a;O>J0q+QNBOm{Izp257UXiIs}cl3!<(|je97vFM2Y)O)zJ>s zB&e(-j*mD4Xg9*WhJ09m92_PZyX?W0hba}TrRi!wuECY~Mgq|C#4f& zjGwj_cnyujO+!?W<2r)QIyQ{ilv76mD-ChD$$$2 z?&A1jqsI;QK;h9Bi#{7tGv^6J5~M@QX9m>W;%hXQFauXlrj09EzVYR$jP7eAay}S%fsG z@xrW3>t^7X9T55%vk0j(d_pJ=Ll)$svNv6`*FTMQW;mzOud$<_nA#&fF0G!VdY-$K z{}NM02N3IJTXXb{vZP_4S1x+yKmh8i?hl`EYC9qP4$GkA82wUoN4WI%^{`-G2sa>^ z_mK1Bf1_wi4rCxwCWxORCrQxl8vVtr$$-&FO*O=to{q=+N+2H|mqGl(K!clT$OxTpZec=0@{FrmeetKR2I6u#^8zZXM$#G7t)cE||eMth(N5>{tOqDGP&!zpaf@RY!jaeG(w^*~)8@B8j} zS595tKs9h0AtQvP_{A1n?eI-Hk^~~AO~cv83si4JEF}`9uGwuYF7Idu>H{URl(RsU zC1p<<#P4wP5xnLBqIlChyHC36MX6nAjm!(C+sFcJ!=;4{!32RL?#o_Q&cya+vByz& z0qR`ps{;~du!kAKq8aPSRI*K258J5lN6%d@bd16%qG&}37F~l7B^6Y4R>)gA#QY6b z^o3%N-rc?HMJ1Th^kjSqlu!ysOaxt06555U0;YbGkKx-Tc@3tAI~cdV-zT7wey7Ju z3xqvkeb1k2r%Z~~x_z&VnnCrr`aZqRLFU6njIJM!*H`>g1R;?qx_3tZ}{!JUIM(q&V|}hsH^4vrn1y3TaAzO&CpzLkSbt7tIN~Olgc~_BIWS3sx4q@buCif}7Tm3|#7I zCbmO2=mmSy#OO8i%&zTpl4U!5s^)`9fehmaMhuszV~t&rx)0WgV2PJgd%4oVuoWHX z;e9k9pzN^;rjXkqJ#r7EuRC<`$IRpZCFUiv3f*aYR#`{~nL_p$mHKgw<&Gz?bJ8|Q zaPE)MW55ELVAHRjDNh@iGVpVIS)J3@cT*PdH~?+;x4dN-jlL#y&U& zEoiH7@Oc}SpJGCaQuI*f&zzn1W$-u5x}kDW;?xw5DXo!Z0=Sh1(IJDoD>MRH#t_7v zU%3r7jhvTHpk{N961h9m73&s$#(wyZ-TS5;^HuR$bxD+^wj{mzCd4~x!{ zXEURm0~0#aaO=jD5PfoXmGh5Ort`}TV7^cE=eV3i_ug=?G$lT>Q zR14aT_J9j;!c2{%>P|moDx#HHIWmcex0ioZL4NURL*I>ZQhCk=NLl{**P6#&Md#|> zVi6_=kg9Q@ec&gO(!q>I68u(MQl<;zx#_JyN2SX@$~j(VSqDJ8>adX=GrQg2&~DzU z!#X5#Xy?huHly;Ao@%&vr5wxihtW4WHzVI~!}?{$BPrrPnVhQr?;l(;k4v@MPlz_U zyV4Lf?Su3u3nvrm{t~mhQxDzsQ(`fBi`}n|HyVZb?*V|R1IE2e%sXVkL=u+L9OjF= zGifF);%`;Nb!AtG@HAzu3k(COo*w4`4RLKVQ|+5Z9*{=^_}pYVol*srzHjJRV9kVv zNNea$+3pt-zwYW#C25%L3_&*VOwX;toK75DJ{%eMJ8xw?=Fll)`_g#%-|*8dPc#Sl z)V}ceXXMLGAzzZ=;9ON-()Z$l^t?;FIYx~KXy$Mq6 zV6c8yE>k)ZAR2cpP?qJv4c7Im**>+}+yD6Gvk;Lsj};fRauHSA>4k!#sU&XsJq{nw z8`~(K((>+jhm7lcJ0?rkkNM2^od(?d-K76g!YlWhjN+*ib}=teh33)NQHrDcpjFL> z?dsAhli$vi6e6xVZijXcI~|$3q`TM3l()Oz=W{^m;#Q#%{R@FIO8iRh6^ZBPm!fl z%xRa=T z-cyLCK>J926*|4OuuBfI1r@x}Qn{OuB!iLS_K3dFYa#8jEb4~6YXbY284lCH$C12f zdAlCeOuVASIRSG8dn2~b{9{y?QqAj(=ttW!_qz?V`_HxVkbSx5EOP3>>|)6LsN{D7 zIY$~pK^pi2kaz2Ozx3z->2Dw>YU~NIXLmnuc3HsPFRC)<@=s4(e9DjIhYOfLtI>%z zwN_!DF&FutXsPk>4upyqKY|<`pyfYLB~?IB`7D31ift;3iVk#b(S{YX~6YjY+Eu72C=q zrY~)L_1>uG9BJ_gf}7ND2sXa+=G55cpaeY;ATC%llTfgCy0Z@O+>w(r2_{s}#M0v= z)8*TC&x#V0dlH6YkdnYi;Lb6i_hIf3zuP{jS$9BEd9gly^j1jJlyyg8hiE>UkKMRtdrC|r78U5kCz z{Q3>3v@bKaD=}aMhUb~m28-m8`h2CYJ@53Cy6^=2BOWvZle*Rdm2S0uj14E_o(ggv zHA^`;j0~<(XF1u1bC#Cg8_bN7VD+(tpVykKUu4+T>V84KvkCZ>Egdt}Uch@Uc4>Uo zOoT0C<87K+zlisg-@FTIdM7$|r(sMfIao7P zTN|7YfJ@CDcPRzOn?dObb{L|oza|z+jySQ`9+^~qNrISwB;az0qATVnz@CEw0A4e} z1l1GdL&6kCyr{qkFu88Sybg}WR|m^sA>2&cJ-@otHG+B>ptuwfW#~G04QOxA@g~~w z_LhwX&?E7^%R^>rZm!Ho#I+gy{aTi!I6&YTKrh^jz+w`?doPQIx}qJ?eXwKItt=50 zjE$y9N~*iovzhKfG!W#EF(Y9Fzz*a$Pd+2Q*@ug`F4|x1Hur!$dt!Xvf)k@qO*tn&YGG@uLXy2X8;k zIrTo}Af}a>60P%%^8ZKESB6E^eqYngkV*~R2o5D((hY*r-Q8W%-5>+fE#2MS-5}l4 zB_;A6pWplc#s}ttYwmOJz4lsbuk)ApVkw)`SNRK;7^B{N3FgDnMobTz@Y&>B@-J+v ze?L919)o#z#J*NQMfFG_`;X9pfm%1*g`EL2=OU}pO@K_y0qlza7hLMe7~2M;$KTVM zKptO`*@v$R7G}Qi2qag6fKL&3PT@4@bB0~%aBk?MMf8NsjA6E@V1{f*VUsDgD0!~C z-2yPM{L(CTr1c`{9V)c*nr8<)S&*}%U^I6F_5#K0LgW?uXu&ye;S5%h&`*&DmjXL9 z-{ZJmCHObTn8H5LZ3Ei!@w#|+2E2lx*Dq9?bPO^lI41m0(w5D%yp~-ZToUL$*&tTW zi#|68nu)i|pj{}1n=m8CJL<%Q;4Sk}V$KHvlx2Jm)g&HR3HyMs$%hp1CJ4?imaUp& z@K7Y>9@lB(aXTe>xa3nF!t#fFISRM5cfT}-DW)!iFaLIF)4&bBt9;;U5(iIyS$KGa{A!PReuA z8*TEe`ek%$`cvb`s#t~61#&v+>GPw5mq$^NY-#d-IJfP$zOT=0PU0CkwFV$Ek+;iYC1c|l}dsdEmzU;l9NxV07bH2{^J_py0URB!u z^y_(kfN52Kx5Y~k19|wfM_`-1WIxa^7mzQ;4WSKLu|FJJ@0a^T==|$z+KMtz7;@90 zPR1KbtjV?09o2aB{Jzc(0@{8&RhgHLyJ}|(IIX*4hgG*KTE6Z0>q0{ILYBX;(*FCZ z@F1qFgh|wA?a49_Q?$&FDb6!3$kM@y1mHtDBldf^{QMO@qgvMXQ}~d-6^8EvMI1wX zesf}^Nxv^n=YlJF9XDE2xSVY2%ZZY<^3Nbe>ZY-YT=a6F)Y@U$e_rYya0pO~;3j%_ zz%EOjt)D@1sYvh0+JIuWlNRx#AaH7tb98TIn0oagaLJ6aB$}Z9Idg>?!$}(&*m4mc z8b{e(q5ljY#`Y)9H%8UnXvC3ugQG2)9%(n{fSP41Lv@Y1$J$Cd2syx2(d^oR&!vPY z6W6f8Lt`>D6+-<4aHfk0yTWU_1Z4Ha3$i)<_~Qxt?zGKe)&fM7B7Ew13(6eioV4#d zgQdwBaAZE`s=(~_pg1v!4U;;P1?)*K?ZXzd%n{E4bvFW0h6sm$q)9Dbu)HKoZEf|qHJ zzaq4rQtoGSV$KtCY-pa0>uOaA6Q?6?o{p`WjD`F1$Z+e(aLu&8yU@lUy{n#Tk;5qE zjE98+^qJ-%jP+7G`~6qDDJ&EYq`S?V|*FzDFL05P{BaA(wZMzw~dWn!fv9T9wzAoR85URKnG!d)6cBX(r1|Dcb6z z6|!*g27}C8v!E!84w z0xMCVZGwH&J};%|Z+IvstW}0K@?K;(;#O3*B9cd$>nZNuB^*^=;aw)G06T&SQ=l)U zV>cVlJ~kpAy(&ugpd2d%%xw54r9be{)NTh;qM$lJ)?mUMO$wzFi|ReX$Fq_dmb6DU z84p#G>>%l9(4;~D9`?3^t&jdg_rdfOYW*sSn-kY!w*rd;^M0dDGlG$+K5=h@@>9$U zAJ7NF)Bp8Rx(UA%wXe*L-aW}fhwE+)&C9l(I;cf^tB(qf?rirJ_f}P{$G}XJasPBK zwW)FVUzw^p3EiE{&hR~MG$*Jd7^|yW$a8BQimhr;mpO>VnrMi_ItC=++U0r0T%$kD zzZTcS>;#`Z*XAepAJ|AyT*y(jQhu~~YShK7qAtedJm0M1TlF~yH$01^G-E-qk(*VF zSOsKHNww+1j6WxZm_(t+!K;^M`YR;lMfjsrI>7b9-KBU$Jg1^aD@EJO%pgAgJTTPI zbdn)&3KB2x?%6Q0l=U`4w~~R~53;E?Y^Gz@&+k4dB3B8T!dxPOdyin8Ypj9N<%k6G z9*e}`EG)Dnq{ZL9s4j^yY+r!p~Yq9{$*0B*GEp`k7gqgE^q^ zFd4iR%HB!Z?7Wx-g&XRF9W*gAR~t5UC-+e-caS%}y6Q&!Es!RVkq$@6?&R?KSh@b*WnT2?IIV5A*a`gShL)ezsQY}mHV?SBybs4KAk{_Cr6 zi|WCeE9qQ-$bA6Jjmq1T^~Tpv%iOAGI`0tPAz#jUbv*t&OW~86E`YL!Ojehd!#_u4 zwNXrUb;DxXpq4*kM19se#}o9Hf$qad|3NA3+Cnt2KtsURH6qEYSoeaAr z+8!5&Ne4F*3eW0;u+?RyiVlPM-h`+kOL#= z6P%p9bvZ4(gCJ~M(QP4#UO!yiR{CO74jHa1`Yz^W6>qmFUn@jhC%iTgL#PYu%mJn{ zS)_*LiAUgr=}*F_j9gGRDh~M5lP4lqGJLd*+V@h&5rk=se$DrdF~t@iQ9}jmxMm8L z16L-jwbP{Oupla0!nOJ#c6w-I<+JMoFSwum!|#gMZT@;*^_@-}RNZGce$_86R6aW` zY934VCa72~{KO406}~v^YX5Tw{N6iubp>(P}A=dSej|D?xBc(aFJM`}rQXE!MbKE6B{?aloQFMfH33M{M`60HU-;rGCiT^pWg84w}2gHG7h z*WOE)^1z4C?Fi$QlDU3^I&#UWA*_~hcXvx9cuqb9W;lBVqz3Hicv)J1;ejLCBO&Gi zDy|TF*cQ7#9z3`oCP~EQ@OZ=!v5=C(-|hRh&aoWxw6b~V5vo}gL<~~B@AqB)Idn*2 z$d{HHkbw2hSA9_?8`lCktLbe%bvF!GyS8SI4Xy#@wNZrcJ4s?Xrc8M+Mla**tALNT z-vxhd=-VuMcO(<6+bzAU=(aorudXOKLfMo|?o{G{`)D`e?Yfj-1#MaNoJU=M*XE3? zH){4$wY2^_E^1|DW@!jEU{$Pa5);gqb4*n_0$y+MuFq(d!@jTQ0EdyajnRVGBNyJba z3t;ezeBD7p(0Iw;>%0#ccys(B?&f;wy^q*utb2sDUt{lr6p=`LUo=!78|G5)cXMoX#5{%#YF;jw$u zKWTc{O7!!(+YQhecyCpmJ-!)-NUjb(@Mop(E+Gd{q~8N5f=o5d9{G?e*BIA2ZIK3x zTMc&3QaA?&BC#@b_qG2Y;FIEo47997&E2o;SLKh=%FTc}S#MbP<=Ry^H6-zR9qG-Ln4YpIEt0E^Mcu5_?ITP0B%eHJd4DrG zzg2*Q+#VOU>z;G{I|NAE+3~gLNyz>G4KgGXs{Bi6+tfEir|ZRMt!s8d$MagFSkh;4 z!!i&FXYmwwC#-4~S_ok!gn^AvnAa?=czBDE&SBU^bsQ~TEn<(|Nfwo%5@m~HbP($# zxH*M%VdD_NF0{_#*Txg&fFrba1TXLsfwo^|aLYLYEk2G7g!iWTsM&<;cW0?W@E=Czu@I|vOpr^;p`c41H|T-AdJt5YEAQY>vb%h2 zzq&)m`H>0E~Mok1ihS3P1zA( zz-n{*@mnZt;`)R>a|LpDN zv)fI1+p^9#eW`S#`yhH>dmcU~_t?Jt-i^OdQD~`Qi3phZ^_Z2W+6GG9h`;R&9Piq- z@vDDW>l$#Om8{NS^uJlTk-OsB+AEsj8Qj+HAw z?LVah68S5NAP-F;L_=k@9f@gIS#M4L$nvmZQx`O;440)MV~9{M*jX3OA#R?TdGz z)WO5_gCHb?^XZ{|o#n5<#iCs1^6J|D^EW-}?Ekw=O);yDY2OYEI|COKtL=^!<CVv(?TGoQaiPVJ@um9LwXIlDJn(PhdGmL$(TUU4 zP1RI9;i|}@gyWBpdSF#@$jOG|FvL{<_!QtmZ9o*BEGaqjkY-K!lp%ua&Lc zEtk+ex+vxHF)N8`1Wp0T95?xB8$h^PL7YS$9r(KmW>HBDOg#1<#cCvlVi;SQ`#j#g zgRfqa8xgsqMQ;yokMUojawM?~f^Fc5J(1nqCzH>Zp=Dhxdd3~q0=ZC19(KQ6E<=o7 zh>@n9VQVH3(0?RSO{B>CZxKAEF!K|y7qb0?P5HQ}T9KPfI<0Et(t1yOyA}b{G%E)BKIf<bG{Gvkk=WXe%=r*@>PkP|X7X{@L^O>C9eO(2vXe`Fvtz zq-~Xml!L6p>-l_PY*lj}ZC#E(%^&L?s-2@|H0pZhnw2*9-+Wvy1lW%T!mz9&i0Tfb z;=V3U=ISZCaQmTH-WM3xGP<>-mf)PU8H#lc<2K5s9cp`y%@j?gLFH zsl4z@JawXh+nPl{Nubkr{mZioyF^(^Xl!bm zsB=Z8M8))kT#MiZ_qyg;a4hjJ)8{vk)1uceaUYw{c5G^Nv;MMq*9h=yGm7ol&FgK~ z@Cp}L>qX80uAJGI&0@R4g`vaaM*fR>*T<#*@jwo8?N6UH_@`tITjJV)$6`A`3$nZd z)Ky-%DY5RjhIRv<@C^rwERo3Ik!=)+#-c~A;|L#&sq7Yq9NCs64GgTp32ahNHA3cK z%kp4NB6>=wB@*ss)DH=0EiLxfY8Z?LxLg=0JGMMZVCV2k`t}BXGy9u5IR9CxBYR}IYHEVMVwxWq=*hKcY6iCm~! zvyufP6f;GvrX~PFIi?if3U?t-*`em`!qy0g6`$%uFL`sB{4@c>B!{9w%LqT2DjhLQ zFA_ttNq}3{vt@*k!KSvI<4B1_Y}Pm|-mQXkX5cA7HyYl>?dL{rkZE$9f;rGhW}t0y z?5@C$^npR9WEDKE?1#dpVZwh!9aSOAWV?V@2l=6u%*;Nv6<|_XYsVhd$HFK@>0HLw}AE7a! z>W9BLS*Jn+`YNZ%y2XNF2=AnG%Xoqk%c1;sO@OV4guIcbe1wn35Ed%;yrNU1btD%LhR^-Yl-D+ zBF7WrsMWFvw826!QsKxZB_4DX@3763Yj+Dd@p=7m(Z-82P+Jg#QublK@jpyoO6M`p z+%GtQ;D+#=(lEAyQFJ_>LhJkdQ=}n9MzR=fFikCH9{fb zNB(O3+`uiV3lSE{rXXntMvWY5P9bDe@_c<*n82^(T<)l*^YmLIZjb{+>kp0Uuz`Qw z07g8EfMm%G-DguH%lXrg6%O7k_SfOu_y7iQP5VNT^d<~wd!%1(h9m?UxQMndBotw0 zuQbK2-k+S^7N;9Pr&rawPx~&!=?tT|xh(Zw&Wv5Fi}BY%VY27z!%CTY7lX@H+Qdft z(^S6x=AR}2Lx|p&D#ktzJi4?UjSf5}VZ9n(tfuU8CGJ;k!812s5Ol?p@QR;# zy@ZQV^`pj_fyHL|{%zJM6Ij`PRuji6HrHh6V1NrZ%bn3(4;r7;x#O=^NzR|r&(@ng z=N0^t?a3%h2H77tvtqn&zrUGa-LaXc9OlndA!S4)p^X zT{@I%ZOD94E`fwN%U&2xKw=nk+6~r=O2jlGxEos`+28?WIf<-ho_aWW?gML)xN z=2xUOF?%bXPj_#oDbqc2XCC4!pt$vxdUxDw6XfX&*G1agQNqs`-{<0>zBh7BFE5I0 zwDXfrv0>p7#4CRr>VCWd)Womc7fVelLZyBs_6>L?xV7(EN&H?Z^klpKRh=i2@1M3J`I?QMh62KMjA5430hzsh(}`VMi!QnjTl&GcGyqsU8J+ri;%@h01VzQRr2LLfq8_Z+y%|R zJnmePc_5sYB|JQ2fi2wi7I3*dx^2kuQ@r$-2K_Q`Md-6c(YnPobV;$3ZW2gEHaxQn0X9KZ*XoZtibE&6qMgcf45R$dks!I5oHu`J0@PlPC&z-u2s9!;b;zC&TPc$?p-8ilklOsrE0L zvDt9XKkwgd6TVFjexn|(mO@bk^J!KW^d8p*Bjk_U5TB)Mj{Hijfu05ONdYD!4p?+| zX3QdJ;kyyh$Dr|CM5h5mkExZMFfO{PsOQ$dw!;U7<6ev=JPUuCSM==6a`8QDP$)@i z3O@lR-*ivNq0Vh_yzgIiH$!s9>2T$t~}nJ#;+wfP@meOQIaR z-;O`zuS)x~n-$NI*HWQdGG1+#$sXlr1afzwxW}59Z}e5X-)$`Gm+jkvm;_b@2EWKY zLQf0fonF;kqH3q`G@gw~!=-8J(m#v>w^xSpU!`KT;KSree4T=;qkDq5K$r}H!~LmV znvbmOf%`mJ(WGqaaD$y;#QCuq3T&IzRV0K02uh$4UxpY1$saXhCGY0s*m{nU_eC8n zB6q)YdUpI_9ED26#!WT9$DQTa7u3`_8-?mO+sY ztxdnD?|N~#c6R>z8$jP{tEi-$if<3?)6J`pf6qjiL}4q}v5JjteL7dLVtGOaeML2h(LdOLeLuQ<=R62B$==Yu;1Y4%$o^C9L7nhzmeXGE^<+hO zKAX6e}sFUs*=EJD{RfhxbVaDA{-s$Y@yr*%aqa*e>_ zWk1ZHM@@Ean5&fNzh}GM`??$|+Mkv(E|=!(vlkjWEfCmQ9xs6;$&~8|{}&JCd61j6 z+8&kLq&#lh*dZa_cdJ{HFH}@7QYknAdDp+4l!PMIl|UFVY}>el^0N>KK|E1#B#$o| z-sqa|Fus$d?HX3wzZ9}hT9`|tnm*X1zysUE&i(YDtUJLbkmp=YSi)%xA1sWR<+U2U z>vzppF5<378GIuLELw!B{Nb@%?i=4>u!;Yq*FOpk>N_gLP)*8-W|x10(W9u6r;M8ZUl$j?YUyQXV@0Jvwd8<#n403wKNCpv#Gv5!K-SxY4%mK zl4~|qM_-H!BH{L@{>8b)PaR&Hr` znu6R+p5YcrVS|5PAhw&eduot0gWz;l6Q?xEg-sfw?I2SqkKrB7QJNzVuw$)(tbLd? zHP^SX4X&q8GGNvIjKVm{ME;7&eQIdNR`iu!21n8WW2QJanZH;2oX`u`E=Y@@pa#;U}FpXOory;q=zh`G|>$coY)`naj+Y!rkm=i#mMz@H=5iM6>1LrXhsAlW20FkVIa=0*5hI zS39McUAKn*@~A*Ym%aMQY!90#E2+y^OaDQIm9$h)B4eKycBVl~d)Vdr)=cS$`}Y*v zjmq#0bvktC%gw3Vi+7i;BD8EFHvRhe=62$s?^ziXr ztNIrd3q^v{zAgyzjr>PD@~GAU6&knCIMu&1x`G@Rj)TGwUbHh6FOZD-4|lvI z2##8`TFg8teE00~^SuY5rUd_bU?X-dn;t*h23IrUVKTLg=iVk9SdwsaMqBP!p{oY3a>_sO=ON`_hY-f^gJYzNSaSs*r!|DJp^iBBAyHe1JrEh{Mc#AaBsnwmr*7lb z(KpJFsXaSRIM01^TLhLZ;-OS+-l2DGo{g<%7oC!jzo%W#rwyIE1r|Lp2l;U!=v~3pKvkwy zO#I93Rb5=1_pd!*#eb#Z=}hHU#09ziW|Zp{(c2J%t`|yW{@kF!;!S~n%G++JzYZ=? z$0UkV{`K3*dAyo160;8c5ku@$)EW$F*zlv!ZtP##_B`X5sAB4h$MPa;%po!0@TB#9 z77GvAt~)N09UZOdKw{@H_<7-HoV^rooXR7evPnsP9;`@G5b zG(p=zuLz{Xv*ud%wfW2V8l?g~3Etu-hLg0P-FM9pIyF(Zt2Lod}A2lQm+@O$&{SQoAe3l=0=|0@1 zSc{>BZWU4-?$jWf*JoEQIG76wgv3Q`VQ01DKgPT`57Plb+l(eh*w?;qPDnXS-EMGj z(D9tY|8EB40A?_w&XUR6mwz(Kq?wnT*J>5y`RoUnQjys-yh;he2EfEWv+Ficg0V_B zGS?l`dmxf|+ELgZrNAC#nD$e192Q@&7GM}-`v)gEFlL*VQ0zjuDJtMFLVN2Ds}X}b zyZJFV{ZHa{ky7Aei}4S(mwqLz;0oj>RS^jk)&y7{p;7(G-;q`B$MMcG450O=&w`Jo zWEIJBgwKU`bIzay8vu0IQ1LOHN2UCvQd4(eW6Y#DI!e#3bS(xXeO4|2Yn858H`Px& zA^>;QBI2VpAsOFo>MjpI#tj`V3v0csWP-EkUC3@Lt`nW!pkY1ZQJOQtz#l?8gOyfO zg-ey+@$klu_hPsR1bH4ZOkX3f5ewV#t#u6Xffs44RgxYwDq@wm)gnK=4}yxSKW{7a z_z|XjkyQg(r8TtKF1nzj##x=zP9Z={|4sNfF~TeB9H&a(=Q`V3Ry7~mk2c)LqADLv z7g&VT9#CG7ey186in(w=Buq{ZUPgA#EdLK=K7zUQnDtovygDuXJX;=)4_8OP0@00u zWvv`r3=5WBdXP<5c1y+yXfXj1Fg{ncR(w_&#h&6`ldGvuw)aB`uf1Fu;1>3=qIbR| z{D+1-s=TcZwI~JP1plG^$!HV3QJ_h1`vix*{|ES`as8!3)WQXVJP?R`j4@i^pRTD} zK8mRpE^2*abjUDn!#2vW3+M}(7D5s&>;E`CbeK%)Sf4SM#D{BsdVh(Jkqbjk4NKw} zP2!Ui{KQd2bYyXMJN7xkkZvaib54Nv2%jr3cZ8|e4-U1c+Q;X!JU%_bTNQIgRm6RP zexe}+6ewnvegx%d8K((RqhyE(#S(b8xaM7Pv7RUTe$qXItmBoc$g(H%M=^E4!aTrx z*gSF{oTr7~lthbT$$6F7Ir z9X9G`vjovwS|gdjG4GSiMZfRa#;lNm$K;Nm@2iRrjM&<<1yVgBLV;pL4=)fsG9qpK6TBMEK3fj!>l7gF!lY5006cIR@Fwqr-%-?8OZ#)o!(74`u2;PU8lm# zRUua#Cc+&yfFXdSgw`?o#%@8s>taz!uPWthB!|oY#$;gdVrPEp)d9+yW0ANm&R=Ta zAb&a>?Q#X

    {p-R!oZ_8)++R!X?t?;Khe<=lB^IY;HZl+OZP0UgtL=I`(G`gbbG` z|EByd4x^-O)J}@zM*$kX(V&g+cNx>(YQ7{iM^U1UbqoTfB@!E0G-{j)8ftX&WSrYj zbVJApWd%c}x;f)Uwr)9ftSEI>2gTj+kMzb}4p&lSCbHta6&C*5&4lK2V6BNyh+y2h-^ zg<7GCS6VYYG+r-YUP!dpt6F>+%G4QEhu-mHk=JMuvm51pyBeKW%vS-3jGyWE*NRe` z?_Bvin>GAegtFWEe4kQ+aO|je;L`%TVG>>K&A*<$oC*Bux9sXXZCVno z@HxN?IK_l*p~-5}_xfiN@6CfK|9@idTZ>KB@yd|-L7{YX_%%P7%uzRo z07`U>9|e$2!LUu~!AK;k&c&+)4nKfNV0!n9ha>c%Aublqmvw8WT<%|IVeVFr1|gN1 z*ccGs2JC&ui6@1v963>zvcN|RX975+2x|9u_}rC5!pL`#;7`g4mc`x1sBf^MG)%z> zJ z7IJ21=~)1ix97&X&fVI+7G2|SpSsS@T-YGVF6cc+CfUz*)*`v|@*Z6N#fZH)% z{nqDus0GSr;s7qe%adlQA2m|Vc#UXA!((#IJr3OIaL zr`|x9Z&{PDc_-At1Rn|oR5Rd_N9SFH9SV?Yi$JCv%O4C8p6IHeamYW0`+0-xGe(1b zzXYb*J1t7W*@uME+RD4AbLgoJhl<1ynJV+2p*o`VQQ!=ie8^SkskL=NNZ2MsJKk;1 zZAWnIk3D4t7Y&I$sv?xb6%1K^WQbwSqS6o>k$iJc6J?tGEi)?}tjc8{lyz{Jyd;Bp z7Otg4aFd@(ka}$LLDLeGIoX!5Er9UTG-cJ@?f9&N@p>yZu3;(XjS7NPqb&va#Y|7O zZn;yxa&z5gk>6_6sG*hY@9W-*+vB;_!&GdJ5#nxY9QRvF1QIL2vkxo}o=LD*R))gDIcV+DS%<)4B8<4<;v)DsjW7*pTbS7BAr9|8 z^ZUn*TvfrcQ0$@pMP0@scBM_MY~e>K-{DLOfZaYeFrtff<>2J|S6s}Gf_Xj&42hAI zn87Bn;fBo_a-W_eQJEU__^T#yBT$SHE_f~V@414w&A=*i)RFMR#MXaYq(mMwuAu`t zq9I2N3FP$>B%X|1JlbY|I0XQR7Ziu9~I05qZh}f03EE9jyIYe{LEYx+qpWhTUC( zTx`5v-`Cneh&F0C3%2C3F`^lNtFK|)R z54#s^>_NH57xk(62zgvnF&L=$%6dMdL_i5%2A5*;ct@`gFuSxEk_J2_ZeCh;;QzuY zz>CkX#<&g_EJVu(hpcEMa*FoAfRi1rNi>lA8rb@_pytx=gL5v_%{G(l#Air+s}69$ zgfk3bw~N=3}z=cf8D+<#uj|stwPp86ScP@X?!?0**!zb zl;+}FxfS`c!j=VYqD-j4gf_&3ff_qpzTB8ifi71n4KhM-5{P}+S&@u^eg>$Ri12?H zHA+E?fmDMcQ{U=vnr*7Z{qn{{Cj9d`jqraM?$Sf#97-UEFg=I(DoGy*7hK*YbAvw7 z^71|)NK@`+Uy6uJEhp`f72(esh)@qU1@^JePQn-BWJ=RhTr|K1zH`roiZaD4uM1gT zw$|*ur8PSO2S znhoBC_=2vX&OgD*!_|(LRtB;t{i*%bkAd5niT-py09}|#o4`FT|2+CByGpJ2^IMr z3IhlrZ9~XAWe4-9iXqs>EJgnOzumMS4uQy-o3=zt?B?=a$%(Pz~dN`hMUxs;nZTK14zwZ9O*0hz#^Zd|_rL0QN zV+%B~G%-7IsVxMWacL{|cpl?|ENsSe`Mj-oao^s!f;g;@;&^}N0c~Cs!Wrc!-qJ=o zGjxB;&|7h|x5}cpyQy7@Gw>mpVzm`Ijsh^8@Qd5G@d1*^L3l>>$jWq#c%L5#nZ9p> z34z2k+U(E(`pBtP@ASbyj^8vA7Rc|sO#*swdneMhWxBeV-=O<5R0FRt28FzXz` ziv$nEl=~v?^{?TTzN$I91j_vtj_yVS^{tBHYNhp5=a;CtgYqZWjuO542`g_sL3=LS zHqTQ;;1WTybhA>zVaHRp7{C&jrLr200vQXnx(@GQqF-SK_a4-u zFJxj_qC!Ne_9%S^QJ>&k15dv9Frh9|ZNB}E`_?8<$zVGE$qr6;Vr%m5wLx+`<%%~t z>+i1IG^Kn-Sb|M0@t2z`!4YhoSxL`=xE`FXz*10X?zSY6>wxj6`Dz<}sc z1sS=D!$WpoH&88_91BC0E2a!QbU4;S{J~GpLaG*9?2;9^byn*H_q*nNzrnX0NAC92 zJC!})vcOZTO)ZvPWL+zjLMJ5Z1Z+#W%gi2i86_SJtB1f*R4H$YCyh!0DYu39A|x}W zvUJVW?w1Lf0FV%59=_|lG#(8VRv#(@auyq7DC%47?_7~lqHpKVJsE$OT<>@=l7Ii= z{n(Pm7Zb7bWiFGYO-afgs=pf3toT3R;ULpKBdB`F_t9fiZ0BX6$eQb*(84qNOVcMLmFf&NVoLj*oOgW4q3?r(NqD%xY+4k`Q`c1yyg_15<@x(4 z{&h6XxgH0Y25O2VNX1=IWrO zZD8=x%Z80cQ-{^jqnmXyi{Cy2z7wM$MTFE?RE^S|bNk?TCaod@n$lT7A-gi zlrDnit}Ft|#gFR>C{)17CnHnYm70G5bMn4gCnbG?=`lY-K_q2&5a9L4aYg2cQd>UK zFO)n0=WF-OFwKm@e8U2QXsq|l`=?!M7mp3qB68oPwX+ac_VLops*b$5Ce`>UlTgLC z^X*HL&%`Q?eZc(u-Bhs-)K)BmOIvFNrfh66W?V*jrNWU zeJs?pBV-w$J!ag@&kj`f5?TLa=eGWdy_|>;h0Lvb$2^)Fm4kSaP^V@}M}%vUdTLXV zA62^r7$}woABn$XzjydW;V>9BJ!EhjGzL?m9NKdkF&WJ{GtDGIk%*v^xJ0I5Ab<^X zi0v{!n6%1((EaK62X7*5_*hr|?|6AZSD^Yp!#+bB5i=?by+|xF4KYhuk+9t~T;uw! zKl}<^jP)r9-(}v8u9776vC>9|EK4FZp_{i(5T6o7{30V=WtdbSJ@cAgRIN&hz^WTZl88mA6$fA~lT#HcD@@ zY9^QNcDrbWvtR_41Spe%Kppr#F)7n@0Scm6zx2Id`bVCt6eDy<>ZS7b-7etuB<4rv zY!S2qDgEteUQEu+^BjEST178X>Z6gX-2m!iGx+gv+7xTl8y$12>%<el z3--?Qj{!B3-FK`!<^#?C2ls)3)2J_iKL|qFJEB4AN5#v)>d@|zo zHHO8S6bWqq)MYH=1PscHN9fcMV4TAXoahsjQ4w`h6ajhkRf}{x6?t9qp(Dt{Zk2#% z!!c2%6XJKbL42}FJjP7CaO2PbTsZd79n`Vp7W7NC#F;N)d*H1@Z2hA7*e|#^CDCFg zXseU~q?0+WOjvWXKn)X)CE<}r+KqR46Nb0bF58WI^eGy7$RJZ9XdVzV-da;_O_GWV z|3rLD0a{>NCAeLYy%2p4bx|f$27aO-LQ6r2(gY7?J@ORki#B2t1HDwB>4~}C=(rqg zZf5C*o~~Zwn(f_ho1!A=ND2LoZ~u2)URrt6|K%(bg?{G?JMLu^wY%fz*d_0JS$w@2 zTGe=2AuSx5nrU9}B2wtY0q8Wq=M(+km(6;kf<$eylm8zCR{S3XrgYqe@5UJ>k=8`f zQVxOh1%5hB-+jaWh>3xRSy3-m&3;Uz{(OXs9d^i2iI6nUiap|x&DJLJHE-~Yxo@Xqv3hl9g|DlXAS9qNbj|>LoQxhZO zt8-eWsfGO@6}gmoEEh$+A+lh1@IcwHGT?GT332H(BzIRH=e=#WnUH=Z*60W2(#v>8 z_l8uHT{TFmw}@#dpBu?1f_n{&rTch!LMhTCeA9?00~TWL;Yni1y#q@t5@?1ob`&dPbnL zoO%6#-ak_N{5fKsW7N*q%j&)zf9^F3h)>+#zpnqUDX)0ZD$wmo0(#{N30yM`8eQS7 zA!E$L|E9$GbPO1|j^Qq2K}a-9C}H18eW(CDbr+^|IZufuA)OvwIV8jMUq(BzfN&<=6$BudFR!C&gS#y5 zCtuCi^iU90q3$cS`yVl2-n;`iz}B?w)ZFOkRozbSC33$0ku<-5Z? z?=tt;f|g!Kdn}$S0GXKiNA0dW7W8gO!X(M;CzSnBbH|4%W0a*4cBzFOeLJ@+?WSqo z75rv0|A4Vly$8y;+0x*a`rSk1P@cX=`D67vb^vtxA#3l_xf~s)mG~85f&#Vq6tO%wM4*xkjUsV?T6pNvn& zptJN>-aM4?`fGbk!jv=w4D5J7f`C%u?tgz}1yeDay#M3461-bH^0KI>{dRuq-Iy&?G0;p2624X`QLtZOk~CuH zJU}N-H32wQk%iV0$|OvVzcZJ++9SSPqBD_8e=H9|#`WK!_W|e5^xB7RDd&QqhFK>? znWPb*o-SisB<-^@sIbvvU}kVmG~Gb<$He~m;M9ac{!vo8fx`p?P(Sj3L=?Sa+0y8& ziTxN9U9fbKnBv@6n;~hbU(zmCbQHl6856a_TXksZhvAQ6rYJpjs13yz@Cfb@FbyeI z791)-Kj!d3W?^ear+04p=zUP#+~+cE^UyZ&0a?tjoKmdIU)vG>aI76fzk3r0Hj9&S zEZxN%=og|#frdw}i%&r7lDG{}lr+hxe6EX!vbJM8qYD05We7A21gpa~#Lc*d;eLn% z2OJor_BR1o#jo(}>^g2<5L}gSIj?YArD>U1bo-0lx*G!IpM=q0g0(pqizl9KUW^y* zw54fw?bLt#H`2~zHw6O(7w6w|u|z)qF=^4`Vk}@(bt63oD z8)V)aj_gkydJ8Jgj?{v|OXxqmL?x2o*!3^d1v7(T6zS1tcsRkP_3jX4V)OkOu=O8c zQ_j#}%Mo6PQqq#ry)-np2U^sS6X;HrkhlVl0h9GAS}>sr+L>N9wUsB`Wafl={L5KK z1Juplk4soWOoQfl?4A{>?0__--CCIfI1asN)xUJ*u;@Ct=iezdp@>8vzC@hyo?rK7 ze8TaW@bPwfO<>ggxda4?LD2c}SHaV>?;=rm3)FvLySnS4)2;3yt%UhoT|Hd&7%d%o z>3)l$(4v~sM3}b&OS{n3J2HULWRgbi`b^>EpGG2-Z=1-keF`HlNi zEoaS(1CW9VkPWT=y!ko#v~(7mQm3k!HMcFHJ9I{Q=6nYRUxqtXly(MB*Zn!oyVRomXdT}R}NgMxGHc6RxzfoUHet9 z9E-OJmh3HgZY6D491YS%#IOz_dP_2Qao^kLIoQS-Bb3*j0{|N z_nr)te`+cm!q2^MIwtNL;0`_#6bQR|Ve-}Ei#3`;?>;_}1(jIuIBouFgG+;M5ABzQ zBzgVlV+yZnU+c9`p@{h8Q^kM}7BuvZF;WV5BaS$1cW%1hvjOY*0aFVjF&Sx~{z?Ea zu;VfxQ>NC_WTssYdkFR`If}S)yV2`KYBDsOE*_Lh=mH-GND$_Izt|C!??$(rG#PXz z|NmJ{&~CBZKcr<;bdI|pPyDS2BYJqLjPdfDOaQiAUM|zCqS7Yr3(YGrP;$+yMfIQR z4V2v+ww6%D9ffnxCX)|RfWELL{UG#lnkh%fc>p>ZA=UwgYqeTa3@#<$7qsdtC|250 z&56?U(tq?eFQztyNH6k^2|*%c)_p%d5nS;Hx0^on3Tr!vIk434b zc>$01d=-@1AL4%z*gsr*{{^1AUfCbL0JRWjXJcQOFMuKxz9w$k@b3gccWtbG;?R<` z;=2Uyj@raI^^i1B4KI8MUX7Mfcm2h`;XhRuRSrg~r(>4}9^D#-ins4MRb5&9t$Uls z!G2C&^K~P=rWps@ACEec{W9)VX>Rk#G?TmNn38wvt5L4r?YKBKcYJYM$TYiY^{o3# z`L48k&C@da{9*pg1F~P92&_Z-u3M{D%~M_79c?eE#|uoH{qL*9UgwVx(tBNWIboc1 zg(NLes(-P0Kb@BO^RrHsU)8$x;`?#gIzaJwCbaJo4T|W!V-O0!uH_kmhn&Ek+JnTO zBFc<)CDqu=Dnw%52nV#~EOaD`!>$-fAoHcJWLf~`VtF?MJ)&O>!+;T3gaFI>|O592^mbR)ha?_s2SWK%FD%gy57FqtuePIRuEHnw6U|r%iHM~_l z)mW@O!2sj>ZaKW*-BuF_m8-Emm|;k<$0g(s>+!G}>@D7obR4+QNIxXhDep?XIOb13 z|7hRFXOIX5P@2Hwlc=g9JV${4Q}sok-`@>jxmRj~K15`uuF-&>ptm|d=Q8fnCj`ny ze`Ola(i*qi67WLUQeD9ZOrBy1u3>vNX68VZ67oYm6P;K|bo*DE7EVAksDEG4{3qI4 z?@7*2wDx;$>>Ww=R-66Kb7f9S#WS?y_#k+SF^PJ0osj6Af1^bP4=96v2b}OyP(+nh$9Vf*qE+{H-*CeMBsJ9FtVQc#HL# z+gW)MyDCxd7b0jdA-+ZE`;~Xhy-@ko*n@^Qu5m)lXU_JiQG3eW_a!meW^}5)VoKO8 zNcrlC%6udrnN&&WM(_92$MlRW5yF0^kNyVeR4}8mXuc5mkNv&eMjL`X6d{Bb1j;m0 zp5Mm!Vt$6pe=zjX{GGalcjHV@2*noqx5dLhrzmhUtCus6i3&z~dC`HkKZsVxqS^Vh zw1WWO{Uc_nK*&;fczAqR>L9n& zVlllh#1VsbJA9W1>&}Ic_6S*trayoODwmdyTsc_gi!+2GnoOSXdmM%X?^Zl9)XPRj z#}o^)l*2bdL7|Ua&Q^jJDxtVWw3>OHlEKK)zEWcpO}coyX~^pm0}dgE{1_sAU@Ty@ z9|9YUy_hBXG)jSk;7kfHFl=c2%Sho3MYioPI1&{`N<&poq5_q{G6e6G$s4K%q_PGr z%Ej7{7{VN7^I}L(TtzX6%gz^fSh%k3C(XCKeShB3{n&H`+|WZPlrAtAiKg`7s%9}$p}^n(wf;o~ zS2yyD#W6O6QZVvpHx|KDJ|Bt;j5ugUk@S-m-f3$vp+7eC{1lW*h$ytEI6A#6Vw&+Y zliGE9i8bhjV^XMvXu&dA)waiDKq$y%LSKhho-l!a`x5p7(@`i#c!Vd3Uc`6yqJTtU zXcq^o&H#fY`MZ0(;4WT@zc(^6gcvTsBqk^{IX>2imZ#%f^>x=p0)qjC;o%5iUhl(! z*|+VTfda4j(|T(X_?Iy&)Fda@@cO;FgbUpnsH7U&F-`@c5{5h*&e2%6y$n;%j>&(5 zxhbRwefju*FQGzX($>f6<;}Agk%90N^v4}}T4FI~#HHfjKpS7O5FuRf!zZo=ogLcl zuH*BjrC1N<BLr0dwrRq3MSK17_gjm-Y)J%-T;RmMWUW9xN#l`#B4LEIR#j`CHQq%K?g zmkhK&Cmgt$(hGvVtV&9joVGEOAfoyBf_|&noNl;1e=(+`h`O>UQQQGwh^I|DOtqr!@8|KL`3otqg)D?msga2~iwCaGh(--t7uV zlC+ZuE;|!SP z{!C&Z9mB;9DaA74@0~5yJmEKT>&bg?iT3BJwr^yXCR2Ya(ikOWweLT=7{%i%NI+*R zNg*G?9%dAWt16MKqLK|+5QfXF9YbZC#)CGt=HBN(hcS{bJfb7yDfWM$sylvgKPmA_ zq_<~A%usW@slM%1#n2pRr+!14SYZxlJ4bI;-VFqO0FyiKF4QoO+%7p2lVEj-?5fiz zkN0ZQzRy%C;#!0DWbx9|ls!21m7}i#V@>~ry# z?={t#uBtrIB0sFxt3y~W7V633yU2uqDF|Ok!c(j|Pfz`&qR|mLEv<7KS!8ZBDw#{1LDZ6<1}}2Ho3^h13}>=mckVf9>4rm5=(L2EsT11Xlb=H zVzw0*6`HjY-s51NYS56i5Z*-}ch6Hm3CnXQq1b6SKLvb?Ki~~~3mqnq6qAE%wN)=m)g@My!1;v<{2M5ia!Qi9e(tPRCxFIXD$%Y0P4jNT*l5c^d? zUNsF349O|%e`{u8eW?@+u5n6#Ppzb@#rwWcvZx_qy#VZ$d~_BNWdddv63Y?d!AG@Y z16NO$laM?@i~_N#O}7J9Q$b5yy?Eah8wPu`;a`;y5Yj5c2R5C|bo92o~O2|U0 z3gh+dVMRawnbv{!p_yTTz5CMxoo~igfuaL`fDyrnh+u*d`KX|9J?!ol5KFp^Yefd_ zNEK&Lts)IDDgwVtu>8=`G)*vG%TkApad&o?uc(^a3=;IombRhT$Ke}JDN7Jtj&|FE zj!&cw$VWOtU{LV`fu(bWkw+=Wp`UbxE?@rGT=PVM&zzu)%(lbTq2evk16ibekn=wB z`n!#i?nScU2ca;Kl`h@T#F8bco%J9T;S{kCZjPdhii~ME6e&@Y-$Bw*38gSIMd_&9 z4W$N=$*&nKQf9f_8Wym_)vZwZTYuXF+4qNIcnJ^Q6!Hp;g0F`~hMp7!s&n5eILNdq z6}3GFkK9yH%XcX5D4xwnaP=s4j6=qYo{K3zzs#Mwo;7RxSih<3y0$!@99uuzVi)^w zI--GU17Ax68;bjBVVB#I?e<^(sHa1MH6u-fPX|!HF~xNL;L*3^Jflf4^{%kf*5^r2 zqVSn54!;GURHx3y>5`6NWv}^{^VP``2(dWizs}BC`tMUit#gq$RO|z_7GP0a`eP6D z=txrte~dST5)Xnby(&>eRtkAh!$*yKxZ|iPY z+dmvJwX_9YnC&95?7MbB01mn&4J|9F@PMb}89c{xw&_2@#&CeOvwg%~|1O5uc z74zGd!MnB90{(4g@QhJk5=WRb1T9F0=XWebIwTPGz;mRMSVprRy%u5*5J}!yh0@h~MK*As+?Xs35sGWFPDD@!id0@osn2Uz#kTv_6dLO*zvdI9=E~ z`9%U%_E&!xIsQ=HIi=>KvY=GpBr;Mq(45j=>06Slb;s&T*NK3A6UpH`)d>~V+ne{9 z?;q`wRPj)kf-O87N?{24p(WV4jgIRGV|hN*>m=kBqQ=u4RcRB1L?Vx9Az{gq)}VACx7| zH5DaA1Kc9^zaQV(V!){67e_Lnxdjk-0VNwK%S6RLjroD3{P^_ZpX1G~v22cwI)_IX zB~XYj5)GpTk6vqa!ePpCmO4T z@eG?e-dKyF6**vk$tcrFj~C>ElEkEf@vodNGX3-4I&wD90F=R$SpIUmsj_|Y@PjD} z{-SNtBIR23rb|noeW6mmVSU|FRpkgBn1A_b^C@^vIXMuxD6Wg#ZuiG)l7+p@C3Ew7 zy2lRR$L{IZgS&2-FlEI0q&D_}u$Cy-=jt`-Af*fI!wtMFzGJz{i;ct896oSu z!2LK;mgC~Z{7UKv@9y;`DwOznEZolbJ{fg?7mow>F!7=78Nf(dCz~)oY9ES8xil1E zS1D-vfo1(+jC!Aug#D5BWIQ31+7gfn2n z?O?;=Q4T5a%;2o>lst-UtOs#4r)6l@dvv#Vq^y=_YPm+St(22n@Tq+WYX$bZ?F2rs@z47! z81E`6IJrOQYr<4vu(&r1Vwp+25}UkZOsruJsBrF1&M)dmTRT5)>s)SWSI%^F<2pZw z-J+GBHEMbnM_4J~^s%m~_n)(1yAW4aMpONAq^W?mjNiHObGv)tKZ)H6BjwrV`?;<( z(NWzm$PH4D@LT2WH?p&e!a4e5-2$#U`DPIQ*JZ|#Vq@Acz;3)m{TU+CntpX18Lkz` z3UB6yzo4`Vv2lIBattzJQl-os+vM1owN%1fUu}0J?*;y57haR-vf9(Zm5jR0-XKpF(kx z8g#--8h5l0-cQG+x!UY3jsGRaJ4qjubuu_&FtuUC`N==GFEG582tSH zWU^iN`5Xu|;td;|B;Tezv3nOjP#WNLXc%qO0neKAMlZ|eEsafsCzd5Xd$|hjq3gaq z!q8-v0Zp5+>S#f8c-W_UhXM#6VSj8|zE;Zuk6n;$^&|IF!$sYH{OBc>vh%&BNE5}E zw1UAiOk8@rO=+q=uqM>E(>zpP71N32!dG~uF@fs`x_ITOd>W+Hy4gmufb5bKZgDoX z$Z!}g$sQPZA|s`X(V#6JaCZSsO+zwRV%QRyv6}=uY+0ngTxw}L$%SW|i1O12ss8e( z5c2RVM$*!#M!ppkoaGZz(emsj9}LupyObFEqhFucF-8-o1jT0!BHc8yZ&1JTCVkfe zw&e3;9De>3WEyin>caS0bs0@_i?n7w7C$q}8>Yq=M#v>V_Hz6+9z&e>(qBSM<{15n zNWQ6%V;zAqA#op9l+y#*v_x_J<$>H(-{x8>uIP51Mz3F zjJ%~f^4_o>KS1i+e*kRD*d9B5{8PdfKlk`za`;aD*w2QHi$M`YJdA74)_{k_aBv8!Jb@ zeFY-HfZ@YXSssi?#2(r+chUA=?19+>p|^1)kN03=k#Ms82=GuZq%b zd86lIv&K>(?VgKUgO=R}d!}o05gsk^dLV_7sY|qxY|!{(F<;hw0nxMM?}OI4gk=BI z0o%_5exzTV-z;69kF9@p*|j`N;ORWJ-ieUu0BP4vS4tbH`a*;5vt;|n|0{P*UgH%BYXL?gJ4X>iNOG4FC5foK@E1Rzi`T_m zu{c{1Y#~yg+e5yoWVG^=frPpM_IHGZSR)QiL38$SYzymv+C!Z!E+je@0k`Rl-e_Cc z{fKy55lZ@Lk*oy0B;!P?lr*fHUcwZ?^b+rI^sB>S`b1+8zXD5oD@y+zRnZP|TPh){ zi|A`QC0uUM`0#@rGCeySksJ3&9NCQLCtl}%XL8dYUEq3l6Wb|6X`Y_BiJ7S_T!Dyl z=`hCggxKEDSLl%r(wF(;&6%9W#y&}%sJ-Kc@i{?*NFxAk7fLoWz*XqCtpO2HLU+t3 z;E}fyTbHK;PI6$3GE)XY8C>rIK?62>2#y7&Q(}u?O=w(YwH|bTp9_5POFY1FL|Cqh zQ;Z>SfwaA$IkY9;m@Oe!EzItCIPtr~A`JpqyA#0L1LIF$(NbcYpujGuT|k-20hQ$* z`CSnO_4YHDlKvx#OKN)KCgOZR-_X~GaB zYQcp^XfpW}Ht~&v!dR}vK~6rP0+V-wzA|2dsIM^S1~%^V$oBl>oPahZj+05GGGS3F zx)y%*pFAuTn6Mq)!YtYFQmg}{A+DloRnnWHGfpy3Adq8UE-)PXPspbojybH~z3_YJ zCZ&FCVqPTwnmS{5st+W}iQdG@k{^}G4}L z16Ib`>R&PbXUF?Naeg_6VqGign4%iAxOxUG*{>gO&X=6*reeIe=Om(7mHxTq#*$0ik#dt$J>AO zL*ue*({aDPl^PpxIUnt^lP}HRs(<1UN{L^#yD_?wO^7P`I&r4h#<163f9+Svu*k`I zVyn}sX~*GKfxVTx_JL7dnj3vw3GgJ|oENfTPDUN)uK*s_3`q4FnL5g&U`Tmwec9=y zdsuZj@>h(q(`K~YAeKAZV0sMT=D zA8Ec5qS@B@U3piM#9@4%kmGOg+<9=awop>lreCSi{=bBBr!||F!&rt!@h*&=37Es* zW)AC%I&yrh6Yv^DCvEK-AX?pZxsCeGWhE97uB3_*dCL|b6Ii$GVGWonl~O*(I2~s# z-2IinPtb6|vuJmOa*s?T1hv6zKs9}xLD}6B-V}-+>YBhDtXD*resPlgNqBklgRi$9 zEuMJJ{4c8OY@{e&QexrWU-x(-J9CoSpqyY@a%bePR$VE&Sa}5^EY^u2S$5PzYZFgI z@XYdv+>eJ7P-ElhDJ!gkko>mSb!NI}zm5aCBDxfZ13XVV9vWyP)%Mz9NjmSTujhUe z=y_{tE~|_6){p3?Su|gMo-&j`xMG*-yV#?-zZ8$DJ&ro4Bl8Vnvj;2yp^t0(^-3N>UHST#Z{s|s zP^_I+f_mSpSMTIqyqVjrYh3bE&~yY{Ii6#0ibMke0gq1}_B=~`k^ zK|4^1*m*;}^{?2x>0IeB2D;8U&~j*bG^K>++vwC6mzb0oyvJs6G`=BPRBF`l#^A7J z_JQe1;tqLel0vVH=y!30SbK6JF`A9s7IX~>o1IM_wU``nVwDf?UqYcII1Nm0p}>t?UZyEIk53-GGi3p-wl27!qjj||}$X_zmk79o_ME6@#lNkQu*wKj;fh8gsto#Wb*+NF+kDxH= zwy?D%;^zF&gRRBAo}_An1N@q@4i@)wMX;HhD#9z$tY{}TiE$EsBim-Vxt#5jQ*GeU zv_2=u2Rt@<*KHfoqrll;{)Pl?ydlr2hsO=V|Na}NZ~a6)@7mDXph$#hAHu$mln*z+I5163plJZ(=kB5rMud z!wwbYVe9lGiQ&QUM2X<^l+oqlmkoMF$|wc*j4%6C{c<9lKE8m%Wpp~^AP zR&OlGqvp){cFENCR80bL&2-J<@_F-4Kc^~nynxWY&7!8Ao8@Qlh>vCG(1_whc3j-q zt2m(OWT~6IQO&OHlI;HC_x9}Ayf`@gfcZ02=swh67c<2T?FowAHE?I>9nZD#0n;G% zGa5|ywWuy)uuBi?gOb@suV8>bJZ+vKTwYu#p_s^v3BMDb7L=TMe}F%FE1ghL6L`Rm z9`=jM7JjWCX#esTXLvNBoH3re?j?iwh@2qeJ{6t}DKcH$~sO-QemXk9_qq3m0yG}V&L&EV}KBr36k?;wI zm%AZyhUcA;&&_(R8JL%yrJHG*BFv!uL}_m`boWWuzT!sc^-Span%;U;w}Qky9sh|K zJGLAm)-QLywOaIZbXd}^Cp-K6JpZsVw<2lb?7F_k&OB{HL$HZ66QKYSIjPrm#KqBi zWhGbvFo{|pyc|8Q9`D}8)#MQ8zPP#kd3sp>rFHhP`IDg6OHC3J{`$F9}8^Q$){J{+1pzQ zc*CxY+Q}?hZiAjTKp*)9^z`5M@p;o;B$v@^Ww#ZZ)~Ake;ev5vvp0VTOFEolm!um7 zBWkV%c6Y=PZa%6X3Wy$Va`G*r*!j!A;O`)AJ_+>9Ky6{}1ReZ9N9U>-4U5M!i`wOC zu)(7XYSQ!!1mVMTUkCAvXT6?7jSYeq7R%3wYUzB`*;^sDqDYR;_dXoIy|G6zh(K)Atyo?rOH z1L0%#$|tykp{kEenw->Bu_as`IG8kf)$y&bl%a7}fs}pKH0@-AYR`zD{&*iZ7e{N6 z;f6RRe-eGT)!g}*aFua{@cl|0(pJ(A#1XrOju;(g`h#`=17T7!$B zOHB5q?+#q~3i)3KY7&x842)RB7(UV{DDo5}LmWUa8=gx(P*G?y_|j917CM*(75Z=e zrO#3|bIvSXLYX%@oC23fUYu}_s*Sx;WAPt`uo3s+aCZpg&piwhJzAM;em9Q29IjML zkACOUR{C!OO_NnL-`)sS3-~qIJ$D&&0g7P}_jn$SaAa%2AMggNn5~o=WpXcyJ+%3( zdl;Qdok2ky9H&^6Mw0TJ+juf~64mlXxlGKSG_-3>_ui(8D~V&n{u3yWPEzsE*erE{z)5LORay)s}iKEZ_@%F=$a#xh$b1slgb)e`23q@L^^18q0LKtWK{$UJpk<87s{tXG0Rj2K4j0IH-S2f(m zs71OBppYPQzEJ^AjRRbKyl2gy>+O9zSU1TbFCaj-)7zneAPQM3*3exfehRqqWbIbA z+-iuiIa!cU0hp9o+jT8muF5M)&z}5jS06oHZ~Lu0n034@*yV#QMf1Ewdl(|d+7W#G z33z)Q011!`5%XWLn`($7Lfj5qozB?2;YR%192jh%9LeotN>eiCdnN)C6j z4OL#PY%22);R+N?ak%$jiZ}{js%gD-ViTZF>AfHrHQeZ@K{fpjGgFj>FS2wg{E@6V zV<6%d%oxOi3MCo(3k|I|$=Dq+{z~fr;KgsY8z<|CItuM?-9jINp-iz}cH_9WW1|1< zun1}IvCt7-ZK*QWHu@V)Uu3)pan5XdmEnDSpp7MJ+v<(^u(r07dHnrFDSPrlL7GH^ zrsd@REVa4%d*ue_7^D*K=PrT>iwBPRC8>qgk|WKArNt1no* z@hB`d#GQGb(ho)2irRPIX$;GhfB3uj&((zq%bXrGt#p;k+(Q|-p27~tyf@nhay8~? z(Tz<6zy7dA^qC)q4=76TJ&35FuRoyarrDbAVnqahK||sP<$^&sun}{JxiG><;Vax6 zVZ=S=AYRN|`u>vm5q-QVZxQ&dD}D$%CXzK~V#epT0Pe+4VqF;} zy;7!#rvn(y?1HCZih5;gYUnzyzeq&zvuuAYpkSsZSaPW>qsr8Y6$WLmf%vH$bG{B< zs(2&5Kt(8+diChNfAH2)nxGno)}e8mNdgrLsa2AhPiVPzlG5_4BKQW8%{15l^<=(M z!A96EK3tTIn9?X+@OaMGwXLS+iLBt73b+R3?(kPV^!?bcJ_jgcf!TpPE#zhseJ<0r z^R;NAdtdg@*2mZA+#S;V{NAM_dFN|WQ8I(x)M9sRcE3Bb4WhFhBDJ239iDTv3vhra zfC`@KAqtE<%VwSJEtEG5Weh2Ozeod_)E8k#1SSN;kv5NQ&`vWmGxG}8Gxws+jSwqF!36E}slOV>gRYd^=P`eSU@=%sqA6w}a5ty+y3LCsVP!zY-nTH? z_S`>qhU6wv1qV_3$2S=?VL_3kgA?{cu;TQCo2XvG7|cB0%`Z+#o43`S5W?W~6VO*~ zSq2t5Aq3`%y12L&4rT^E*iZFV??DcPRZzO_fj6;FOY}u4?vf&Z%HM3PUPfNI0SAIO ziM7^&y`*MWy&vUv@;LbX=@XF6Hl#B$nn$_H%G9yBhN|+Z6NPbIsSGI?7d@8)l5QcAnClpJAwEDJ`H0E>*eq`Fx zN@ls;w@hSn$2ck+wcjMZk=z|#AgD`0{w$mvhEQ6LZGK??sPXelOP44eO;0=;%6VltHP23A_qCGOB~4XLYYB@7!} zm_}}&MjL1Fiya85D#1^=OzyzjhP-mv{_0)Drk5Uau%!UJJ=v|v0P8pvVY)C1e&66D z6~S~_3XzZ?K&=^+X6DGT|60tAd`u}=1Us*lCfYa=3FC(6@;Hss@MGu7KxCrM&ms3# z1zWONw{=tEpZ<$X*OR5I6TALv$FlagpZ<$aSJo%S{RZ(nYu^T~X+{p6P-MsuOZOyK ziEUxkpC%yH0#HgNr~sSfwF((r>^kSR6?kyJ{^r4}ujj2IEpxQ(bn@0pmqWeL$#v~V z%v?_i;zc4u2|kTPoP(f3DK5W@M(jnc?^S;5z4Rx` zq*k_gA|recOa7RkicW(j4N5LB&mjwJ9%TwkfFm_9?udKg&w$&<*rf_nj$3UKqv6b4`fMe`RIjsMsBmZ;8y@ z@ntfa<#X6XS(i{|OdC^A1~f%pFgvqYtXk+Jw|patQ{;xMvvFHW&0AZr+TtgCaDP}H zg|U-2!V)@u)j9ibv8-!Y{1+|ogJZ*=ERnd zS|WU6OFs5K6vrgUt7*k$97T1HS}$_@@K0%$R;sHWvWF^h&3pY>6$P{R7wvxDZ4`yU zby!*3v)jKkbLYDZ0I8XcX|K<;^RS}j*>`IrZY+9ZaovCCsw)AW(*-ML*C{VxRP2p` zC}CDiPFnG_GcIo7V8@BYOaJ9X{j2U@%i{apdmz{9A_HVdI-&D7IBy1JW=$S-fkn(; z+;Yr=ul=IiqtNkGs9W&$P-}r@EFI7yZMdBu5syTM5QL|LR`ABNwg+<{I;Uk?i~@SQ-ywDAHKO7vX~qZL z#6p)~{UHe2#g~MzN)`8F4QEiZDv;pL$sAko5Bm0e-pv0QZ2NJ1w2se>!*^3n1pbkPQK+9ytLWz?ER zR)*l&lhO@sHh)yxoa2g@tQfn>)7B)7k{Kx$t5o2-(Mf%&)U}UQ9`+QtQn?-4 z-RXyaJiZlex8=hLy;272(}f!ww~?^@D<4<@8td#AE{00Ff7SPg z6ikw5bbraicO*9~OrJfKcyf1ls2%FqZ+K>V?LQ3szEY6_=yZ;(UaCKTHNA0mXFZ1QSgNeMH|528qR2qeT0m7_8Xl8Wfn~5mWj*$H)^s zP;CbPvc_u*5j!VM+u3>H27geVF`+^HZT`)u;HTHxwg`cDBuf78J7Km;1dbKiYVc{x(c%{sxM86blyZ{V$injf#g`5O-IM_zg|eIY%WaWj_$M`WJA zLJJYB=tLqVt&xDHk#iyZeNlL++RCh#Zr|lqxuI0)q4};W_okk>M&y-y?JQfIr5&fY zWpli%Is%rHkpz3LNfQZba=X#AvXTIE{s`@qeU@4ELxzl}fGr4fytb9~xo&X>bz9})}AW$nPBn~h|x>ES^lhfBvf~nTBI8y<|)moW_^}!y1X75+Zoone(qexE-vu@M5-Qc@5 z!7Q5pxdLm|&MZb;@xiw2bV%FwFWzgbs$M{u^TU*vjZm^}mbjaiTk>Po-31_Fekhfc zGMTpc=5$wgsiM~82c)kZ*xCQr>B4sWd-uv5i2qYx5uqjJs{;G*5K+b;qEaYwo*uvl zCiBUc1-bba z_?EzJc9(q}=D)hYzcvUYv%kW-%5$o5Tb3^N8$L1aEnE3GEUa6v+@+=)&0phoy>LyD zad-gvJ~H6`f$>{H@Ex#9l|71=-(JCs)@O56pgS*`AwHLJrdZ2T87bY$yJ zFE-RQ;C)oUZMZ+1{kiN|Xw~Nz`@it5=6J)?;`ECoB8(>QiU(veVf$?N=i`&nta_!N zhTUT2$zzPbfn+E2oeJsq=s~#|PH%4*BRcnFDn4yG0~Y_2jUklH9#N0Lt3~Elc3v)( z7*mpPS{1=fqdYi;Gu3O~JP+89KggjEQ3fZVZb7SXnc=h537Uez0}mQ(N&=-`B7Zn; zTgZ(rE5Qh>Zw#rl=QmhyF`Gj63=ge;4|BOGT>+?2%AVg|L37UyR24#h!oWKzC8TWipFVs=k*vra85g&RvDLeR{u2Y z_!B60OnNtE9c@|uZh37T11iVc7aow^g)0@k#;>}R-x01{n>xN!z45yePz?T0xn$tF z^0Mst^FvG0f$;&4NIaZ=?~sE1?-5D1BLMX=5$$3SK_Tbu!i)5MfJwik$I3}3%PhHgibwS2YeGQ(#3f~4`77h_FOIF zs|_`1#RxG7&_@bOHA~d!84k-|Hu9P-m_5M`@k|#=yxF-36h->U4W+I9*h9u`>Vw3c zaaF`P&THmQXiOTD?^#ZM2CY%iK{ftNIlP24T}b3tiO04rgQFaw9*FHDW^h_>$F_8$M5g2ah{&mj>{Rxc1TpCF&EYOV3lo zd;n3zSTs)ghoM^>MPw!H_W4eLlH)d+SjvA29u~V7vGsAS=fueFP}wK?^ZY^PF(w6c z?FnW*dRVrLQheYEKIn!~1#a$kJ-e~Lf_KXCi(DYQN`Vwf8#WW%hoC38rcr$^K`i3X(&U=#vZs8$VhVem0?{t zgisJ2i3DZ>FIaw_wGe+$(Co@`wst-E~z*3q@5Ii=nYn=qJq)>=&iJ>X0B_W;@M!yo$~6lcfZl34x*po8$v zdVh%?o+1)H`<~J00aY0cL^vXmPf|DKuNJMSQbIut`Kzpj&A4+dlPWO4QKgN%`eU>yMj4ISE_i=qz&~OwrQdVIfDv=YSohu9U5eIr(Pp zm~Q%5JsD@R6weY`(llp8V4&tlzV}1J@A5485(s!2yYw@xPrbMAFX)D1E_fL>pk)Dp zZo+c3KqU}B{lQ|HIQ;wDDG1$URan+>-j1!lg|)FJ=^YtaHMV8NyysWu$7bis5_bF` z2Wrr#!iY3-ydprkWM6vn$jG<#p;Hjq&6WL1h0-w9N$S7k9BmGMY+hcsJE!f1`=`&# z8chJfW7pH`=~;W>O;lmSO2WMVsiiJ&TsO%VdWISjA7Zn#o67z6Ns=g=o48B5H^sAG z_x?0ZX`ip4m3bJ&agk2mWxciiJ>$$62u6K8LJN@RJc|f)*F&C?&uPikDRcdfvsnhy zUpl@ZVbsr;w@NaPYSMvz+=o(x^{s^Myu2S4L0MjuBp+oeNG^nYaf=L| zgj~wE4*brDrH3tmFLjN;T$e%KW|hge=vzvk6HfpzCV!hTtsRie5Jb6I0G%TPyQ;x2 zWzW@P4Y@$0|A?OHuNPT@M+laOkI6OU^YPU6fFZJ{9~HHF7Pttg1&d~qmsBss=($Dz zc0-j~2oNoq69~XB>IlB>^#U#_hm9wvm3T~2-2+VYMv(N0;62!_B8TukOO9qTed0My z>o9(geKRdXD-jQv$Jio4J<|noEF=S!fr5w zF@RWjVyblXt~=sTUJ;XMVLv<^>@hbVYz<0$S}6yN5BZ#D(l8bSUwc6Ou<<5sb=m0* znU4hK4CQ+vUZ!wlv;VKM>ws!%S=WIeNQa0N3DQfD-aDapgn)=3q4yT5fWV=H(wiKL zbX1gHmEJ=aklvAwbdV0d%{}Mb`|iE#z5CwI%36D`?97*$fA&BB%>4O2kx5Ad%;kti zk4D};N)d}!CPCPeuNld*AwwlSqmztN`oX_N~sGnXU^$n35rKY=J9DKbhs> z$p|JR`^5sv#T=c7;Pwp7bq7#haQnn--Zu~$(<}~Sa@g31CYl7k0E_P2C{H)~p$)p| zCIj(pV#QKGO@IS~D|S1|oo?OEUKix>NDU9Sn%?SDeCfU7p|NI7Ell)FO~&ws7x%ao zxRe0`dC!n1rb*6&Lcgvz{V!ei54^8l(iLoMLPsOJ`W5e)@37h3Dsc3)lrpYq8vFUs|;`5#|x2iS&*_L^)}I-+Go6&?^#)Df+O^`B0C>iY;GY0im(@*#)F z?dkQ2wZIg!dTD@8Vjjs|L;JKk??Wi@QT@!o#aR8hXMIvF=U=wF^7Q97Mh>QbOO_uo z$d*bPBP!#HNgf(#3jmxhK<{aM-ctpr#4vq*5PnGg!Gg`(ZbqkvNJ%nCV$GTIe0)pT z=1|_n_c>rPSd?&NRq?r8~fs{ROfn^1G5s7R#zIjZ?;i)0)+2?C$*1snD za+J%WqdQ(ufN|*S(p|1+Lvb8kDF>ntYxv2O6gikKi^OC(suLFj9w2fTLSu$8+Scih zRn!v9r%9d+nsm}X9Un*U(*DlM|15}~XqdOa-{Q9rU<9ZLNE}of9v9x*b@rbwD%nk& z+Kc4AuM<^5vk;>gXER)6U^9c~=u)2rH3Ntih0}3E@ znuJkHz1Ck2Aq)deKOB8QmxBp?1~W5fApaQpdi6-^iW&d7bV-U}@N94%AP~=0H-&}E zs=8i)RhpK!_G94sM4KNVkzTm|Y325^_L($Fr-P3IDP{#zdL$&rnV!NIb$RkNko!l} z@i&zcPmas*ZP7Vz+0?`D%RWuyy{fip;nQjgK900rFhvILRhusc934$pi36TZ2I|y7 zwsLfY*SFlj>&25bng^OuG$EVmIpU$8FH}*(Bnz1ac`6Hv3K5;Q%KGl@S{~5|CgtCU^mR( zu865%`N5BI?1HaJNVO#Pt3&nKp08hZvd#GL5n}vB}=Y}xr!q=&gfdhz>1vs*uvk^jR3}~C)R@uba6tJ?9%bBQ*WgOSA zg-^qnlsCSwp$N_jCwnWin4?8K?QqL^=(hyDK*ke}MzZ@Ll>pOMxrjmX{4*+*rA$VU zt}!K-#Ptv8;}%Or)rn30H&QwC_%2ujnOB$p#Tk28!(4`1;$v_%8Xs(0hMl| z+~V+j0C@jR|3%HkS zW08ioHL;g!hJlT|4u`XfEB2OJ_q61DJ(Y7;Eo2*!U90x<19l14dMg~x9?Cf*jDWMc z$T#_Zz72#CeoB?9VJbu)7UC}R#|=Jnh$UdmoNL2+5be=sBmvwnJxZN!2k`S&s#~EE7_n-lhCxB&VwtwO& zp9+lTMsNXHZ291e0Fcdear>0C{X~2y>JcB#(g$KctW;E{Kx|7&!E83H=e|aD{=1_= zd4fxxSh~ub;z&F|?{yhfH!cO%!)5e)^R~AIFGPZV!5~>e)v=Pul5vbeQ|yVMW0%g; zqEXL%43hhloVE$z(nPlosT0wP;U5`tQ!I3Kp{Dz8K|5B!^)#e0E#uB5osj_iVR(|R zjo&emy8$(oloXeEoVCT>?R==9*Bq2RsWt#lK}7wOd1E7)wp8I93n7IfGG*t=3exvy zp3}v1UEuJHz9XQ(cP({R<2A%8$_{5&Fj>A&DcNg45qN~3_mtQkLIC`W^|L0?z>VV? zD8smShMrG3Y_1zCNqeXJ*`Q~$9(d9IKDXVfWN(6@l8h z7=0Qp&%fY)NOiYDqk*iodsO+&oWwNS!qwT&@U`O!X{j6MBU6``6B2c=h%?CS?UY28 zzVLh-tOL9s(+%{rvT~WY^4DmLb@Lg@-G4a4@7JENbO!DxZ{n%Sx_3;*zfIl<7z?kb z^mSgi(#PZ~ZBwaPI>3jKu z)=kt6dDOX=IbFU@`!~mDltnTlGM5Wq^}Jm=I8P>Tc$+~-5qKjJ!wsyQZezcVc##8> zhR{TW$FX089eN;&-Nu7-@1R4`hBz^hN(f3b*!%`M6>bw~1%6cU$qX+bR{MEd6VZEn zxX0qz+3}hH+&j#2#q>#P3Q0`hN|a4qyG<6PEXI|<8t1))RxPSZQ+eZYa~}U00p?>Z zazQ~xLpx~qBug0b^#GMciOSdNOzL~kGDr5klkhTVS;cxQ6Tm18leyGqrw%npXtx;J zu=JcuF3(=s=!&Eh_gG++TpSN5y7a?jgzL^3iv+C|Zy3i( z-bV*y?>86jEsNs-)}!Y;TI}$XhF$fx`f1{$9nbQcbz5Qa=T5sJ(NSxq)zai155yX4 zhyk+6Yx5n-A`FXIlhCF{fD zz1o<0h!}$jvcJE+-vSZwCGdL7db!`gTs9jK5Q)~DXE9#{ZKe~&ybjFU0znq88I^_k zWRHN5!ui5kb$WO1b? zV~)Qb!w6=OhBq#>B4o$ISh3ViguctpuPvRk^a2&Zo zT9uKVhx4UCX5%h9JekBH!oML0x`Rdb9Uef%8OV$BRmUZ0lqdR?Ids1_Lu&t$1r}t) zX*LqT(!rqf(|b;Pf0Q$hr9HDFMqRdvGXsK9UM6h4=f_7*AZc(%`O<_>i{X2j{l!CG zpSO8S=7|2~#~qt7g)h+omQs-eNIQ|QSvseLI(`#Ta`DZ_lYsx z6rS+KOK%oT?h?B+B<%^E2HmqqpH6(DF04udLT6t<9i?N}+VodqnP)~a3Cu0@_Ms~G zVfGRimEj`r@K07(PqbxJ@@r?+@Pyh{$R)!O#v*H^5GOjq1>H}HM=jTYJW$wmh~6-! z!7Hn&czAq_Yc3`$Jv9VzZUVL5)iKVd0rVZ_u~GdCJTY&qLm{XR(g%VK8WWN>$6}21 zGZdf=@;NYybd|8ccH(|k-WWVBj376;C3z5HE^_#6FOeI2>Yi!PaHYwl@}KS^g>HH@!K^uKm3_@ID9Z zHwuoOG!j7+c$!%0?nICbNK}Fg|A{$n}i!J z`s)-#*8en-aaUuR{OHs4ozPPPjHbrUZT?Pd6K$10yvlpboAgGC?7}83?suo<1gDGZYPtf7{ZtI` zdbL&W>WK}$2d^ZR)~oh03u{p?O%DUWCZy`w0mlgSY@VD?9oR#xUOlVKOSOFzf z?^);)_~WJ)Oqd8~Dd`9MEM8@oavZK;MFnI0eiC@-s`>F9E(?V8JtosLU8m<+5mfuU z`o;Nv9I^r6vR3y`7vz!@v9pSkTFN>oxXY^iWNpV`ZuifvPJ2GgzMu?af+EOd2q)hb z=IhA|P8FlsWXDh`6luD6r9mEp(n>^?3^jr1tCV-4vIHre*rK+mN9HMbiv?wa=0!7v zW=)&q84yfPZhT>Bomdry7D~JIf+}c!@kxz0Y`gBoV|2TK-=@@T!`3_zdU@Z`>tx?n zPc1L{^I`na?(#Q_t1r9nm)2$nL|v}f-PWDZ-nl!y$UXhmd#&#`H~z`G?p5c{7teKcltLs=yog>sh;4@r%dscVRjy@6ky>-(Po(oA z14?wI6Ob8R#8ohYxuRy{^_ifV7O@%j5X!)gYivgzBWDT>zDx3Q%=oAl?rVJddQiWm zJ81C9&&VeYWGOIkTUaRd@)X87K174t(r@H}fEb>R!LF5-RL17lyZN%&aW$Hb)V0P_7^jp-HKLQ;H9&(`9`@!T!+Wee$+|&H82()==P{} ze5XYvtu}xQoZr3)Kv0mIv!kWErI$OGyO;YDU0e{j=d=Pj`OMbX!wVmTfpvlb0%4H> z_v!%m(cRvtrG|m|5d24hb0vb>z-NCy@IaJ*o|AxfWNuI0Jj|?|9V{$exu59X0fAlr zz2HB}yM?$5dKKfZim9iN@E$<51%p5wf0hSaf_4Js|EWAPXE%3eCu5X{nS-slv75P# zv$KPnF&_^vpRo_G0~gBG?r(??|2h4Csmi}1Ci*?qmAeZ91rYrM5r6^^wK4yD4gLe7 zn+@v!ium5%fk6SNlmphkB3=Lu`;RvJkBDCXS42G%u8bc;AkdT_>EB!Rk2)3*{67#a w%&h)*#Q&+I{n0P}u;a-bfcQ_{4(J)T-R|G)bO-zPH(~=XJHQeO^S4+31)BZv+yDRo literal 0 HcmV?d00001 diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 4d8c6e2a8d9..d5a757a65b3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -37,6 +37,12 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static java.lang.Thread.sleep; +import static junit.framework.Assert.assertEquals; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class FilesIT { @@ -1413,6 +1419,62 @@ public void testValidateDDI_issue6027() throws InterruptedException { } + /* + A very simple test for shape file package processing. + */ + @Test + public void test_ProcessShapeFilePackage() { + msgt("test_ProcessShapeFilePackage"); + // Create user + String apiToken = createUserGetToken(); + + // Create Dataverse + String dataverseAlias = createDataverseGetAlias(apiToken); + + // Create Dataset + Integer datasetId = createDatasetGetId(dataverseAlias, apiToken); + + // This archive contains 4 files that constitute a valid + // shape file. We want to check that these files were properly + // recognized and re-zipped as a shape package, preserving the + // folder structure found in the uploaded zip. + String pathToFile = "scripts/search/data/shape/shapefile.zip"; + + String suppliedDescription = "file extracted from a shape bundle"; + String extractedFolderName = "subfolder"; + String extractedShapeName = "boston_public_schools_2012_z1l.zip"; + String extractedShapeType = "application/zipped-shapefile"; + + JsonObjectBuilder json = Json.createObjectBuilder() + .add("description", suppliedDescription); + + Response addResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, json.build(), apiToken); + + msgt("Server response: " + addResponse.prettyPrint()); + + // We are checking the following: + // - that the upload succeeded; + // - that a shape file with the name specified above has been repackaged and added + // to the dataset as a single file; + // - that the mime type has been properly identified; + // - that the description supplied via the API has been added; + // - that the subfolder found inside the uploaded zip file has been properly + // preserved in the FileMetadata. + // + // Feel free to expand the checks further - we can also verify the + // checksum, the size of the resulting file, add more files to the uploaded + // zip archive etc. etc. - but this should be a good start. + // -- L.A. 2020/09 + addResponse.then().assertThat() + .body("status", equalTo(AbstractApiBean.STATUS_OK)) + .body("data.files[0].dataFile.contentType", equalTo(extractedShapeType)) + .body("data.files[0].label", equalTo(extractedShapeName)) + .body("data.files[0].directoryLabel", equalTo(extractedFolderName)) + .body("data.files[0].description", equalTo(suppliedDescription)) + .statusCode(OK.getStatusCode()); + } + + private void msg(String m){ System.out.println(m); } From 06eb9b623bc2e44838fe325edfc9ce5b13dcff57 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 22 Sep 2020 11:22:36 -0400 Subject: [PATCH 044/113] This removes a few redundant imports that Netbeans must have added automatically to the test class (#6873). --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index d5a757a65b3..a538cb54f59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -37,12 +37,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static java.lang.Thread.sleep; -import static junit.framework.Assert.assertEquals; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; public class FilesIT { From d01ca830dc4e3df7719c3f1e85a95e374ed18e47 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 22 Sep 2020 11:29:02 -0400 Subject: [PATCH 045/113] match style of other curl commands #4225 --- doc/sphinx-guides/source/admin/solr-search-index.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index 172db56c866..d37b7eedb26 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -18,15 +18,13 @@ Clear and Reindex Index and Database Consistency ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get a list of all database objects that are missing in Solr, and Solr documents that are missing in the database:: +Get a list of all database objects that are missing in Solr, and Solr documents that are missing in the database: - curl $SERVER_URL/api/admin/index/status - -Remove all Solr documents that are orphaned (ie not associated with objects in the database):: - - curl $SERVER_URL/api/admin/index/clear-orphans +``curl http://localhost:8080/api/admin/index/status`` +Remove all Solr documents that are orphaned (ie not associated with objects in the database): +``curl http://localhost:8080/api/admin/index/clear-orphans`` Clearing Data from Solr ~~~~~~~~~~~~~~~~~~~~~~~ @@ -95,4 +93,4 @@ If you suspect something isn't indexed properly in solr, you may bypass the Data ``curl "http://localhost:8983/solr/collection1/select?q=dsPersistentId:doi:10.15139/S3/HFV0AO"`` -to see the JSON you were hopefully expecting to see passed along to Dataverse. \ No newline at end of file +to see the JSON you were hopefully expecting to see passed along to Dataverse. From 3a579aaeba157f402de795610724385805eb0b68 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 22 Sep 2020 15:56:52 -0400 Subject: [PATCH 046/113] initial commit --- doc/release-notes/5.1-release-notes.md | 151 +++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 doc/release-notes/5.1-release-notes.md diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md new file mode 100644 index 00000000000..25292120b9e --- /dev/null +++ b/doc/release-notes/5.1-release-notes.md @@ -0,0 +1,151 @@ +# Dataverse 5.1 + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +## Release Highlights + +### xxx + + +## Major Use Cases + +Newly-supported use cases in this release include: + +- xxx + +## Notes for Dataverse Installation Administrators + +### xxx + +### New JVM Options and Database Settings + +Several new JVM options and DB Settings have been added in this release. More documentation about each of these settings can be found in the Configuration section of the [Installation Guide](http://guides.dataverse.org/en/5.0/installation/config.rst). + +#### New JVM Options + +- doi.dataciterestapiurlstring: Set with a value of "https://api.datacite.org" for production environments and "https://api.test.datacite.org" for test environments. Must be set if you are using DataCite as your DOI provider. + +#### New Database Settings + +- :CustomZipDownloadServiceUrl: If defined, this is the URL of the zipping service outside the main application server where zip downloads should be directed (instead of /api/access/datafiles/). + +## Notes for Tool Developers and Integrators + +### xxx + +## Complete List of Changes + +For the complete list of code changes in this release, see the [5.1 Milestone](https://github.com/IQSS/dataverse/milestone/90?closed=1) in Github. + +For help with upgrading, installing, or general questions please post to the [Dataverse Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please see our [Installation Guide](http://guides.dataverse.org/en/5.0/installation/) + +## Upgrade Instructions + +### Upgrade from Glassfish 4.1 to Payara 5 + +The instructions below describe the upgrade procedure based on moving an existing glassfish4 domain directory under Payara. We recommend this method instead of setting up a brand-new Payara domain using the installer because it appears to be the easiest way to recreate your current configuration and preserve all your data. + +1. Download Payara, v5.2020.2 as of this writing: + + `curl -L -O https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip` + `sha256sum payara-5.2020.2.zip` + 1f5f7ea30901b1b4c7bcdfa5591881a700c9b7e2022ae3894192ba97eb83cc3e + +2. Unzip it somewhere (/usr/local is a safe bet) + + `sudo unzip payara-5.2020.2.zip -d /usr/local/` + +3. Copy the Postgres driver to /usr/local/payara5/glassfish/lib + + `sudo cp /usr/local/glassfish4/glassfish/lib/postgresql-42.2.9.jar /usr/local/payara5/glassfish/lib/` + +4. Move payara5/glassfish/domains/domain1 out of the way + + `sudo mv /usr/local/payara5/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/domain1.orig` + +5. Undeploy the Dataverse web application (if deployed; version 4.20 is assumed in the example below) + + `sudo /usr/local/glassfish4/bin/asadmin list-applications` + `sudo /usr/local/glassfish4/bin/asadmin undeploy dataverse-4.20` + +6. Stop Glassfish; copy domain1 to Payara + + `sudo /usr/local/glassfish4/bin/asadmin stop-domain` + `sudo cp -ar /usr/local/glassfish4/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/` + +7. Remove the Glassfish cache directories + + `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` + +8. In domain.xml: + + Replace the -XX:PermSize and -XX:MaxPermSize JVM options with -XX:MetaspaceSize and -XX:MaxMetaspaceSize. + + -XX:MetaspaceSize=256m + -XX:MaxMetaspaceSize=512m + +Add the below JVM options beneath the -Ddataverse settings: + + -Dfish.payara.classloading.delegate=false + -XX:+UseG1GC + -XX:+UseStringDeduplication + -XX:+DisableExplicitGC + +9. Change any full pathnames /usr/local/glassfish4/... to /usr/local/payara5/... or whatever it is in your case. (Specifically check the -Ddataverse.files.directory and -Ddataverse.files.file.directory JVM options) + +10. In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above. + +(Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`: + + `sudo chown -R dataverse /usr/local/payara5/glassfish/domains/domain1` + `sudo chown -R dataverse /usr/local/payara5/glassfish/lib` + +11. You will also need to check that the service account has write permission on the files directory, if they are located outside the old Glassfish domain. And/or make sure the service account has the correct AWS credentials, if you are using S3 for storage. + +12. Finally, start Payara: + + `sudo -u dataverse /usr/local/payara5/bin/asadmin start-domain` + +13. Deploy the Dataverse 5 warfile: + + `sudo -u dataverse /usr/local/payara5/bin/asadmin deploy /path/to/dataverse-5.0.war` + +14. Then restart Payara: + + `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain` + `sudo -u dataverse /usr/local/payara5/bin/asadmin start-domain` + +### Additional Upgrade Steps + +1. Update Astrophysics Metadata Block (if used) + + `wget https://github.com/IQSS/dataverse/releases/download/5.0/astrophysics.tsv` + `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @astrophysics.tsv -H "Content-type: text/tab-separated-values"` + +2. (Recommended) Run ReExportall to update JSON Exports + + + +3. (Required for installations using DataCite) Add the JVM option doi.dataciterestapiurlstring + +For production environments: + + `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.datacite.org"` + +For test environments: + + `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"` + +The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example: + + `/usr/local/payara5/bin/asadmin delete-jvm-options "\-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"` + +4. (Recommended for installations using DataCite) Pre-register DOIs + +Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. + +Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. From 77d89a26854efe768e6a75ade5f99a6c5397ff84 Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Wed, 23 Sep 2020 13:32:48 -0400 Subject: [PATCH 047/113] Added installer setup-all log to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ec135fb3618..b654d5fd0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ oauth-credentials.md /src/main/webapp/oauth2/newAccount.html scripts/api/setup-all.sh* +scripts/api/setup-all.43167.log # ctags generated tag file tags From 0b49f3b030e826b8642ac278f3a6d390ece54db2 Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Wed, 23 Sep 2020 13:36:42 -0400 Subject: [PATCH 048/113] Added wildcard * asterisk to log file name --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b654d5fd0b0..4d08cfb2257 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,7 @@ oauth-credentials.md /src/main/webapp/oauth2/newAccount.html scripts/api/setup-all.sh* -scripts/api/setup-all.43167.log +scripts/api/setup-all.*.log # ctags generated tag file tags From 789e5ee154ca37764ae74b1d8472d7075b919212 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 10:32:01 -0400 Subject: [PATCH 049/113] highlights --- doc/release-notes/5.1-release-notes.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index 25292120b9e..948756e4305 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -4,18 +4,30 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -### xxx +### Large File Upload for Installations Using AWS S3 + +The added support for multipart upload through the API (Issue #6763) will allow files larger than 5 GB to be uploaded to Dataverse when an installation is running on AWS S3. Previously, only non-AWS S3 storage configurations would allow uploads larger than 5 GB. + +### Dataset-Specific Stores +In previous releases, configuration options were added that allow each dataverse to have a specific store enabled. This release adds even more granularity, with the ability to set a dataset-level store. ## Major Use Cases Newly-supported use cases in this release include: -- xxx +- Multipart API upload (Issue #6763, PR #6995) +- Administrators will now be able to specify a store at the dataset level in addition to the Dataverse level (Issue #6872, PR #7272) +- Shapefiles bug (Issue #6873, PR #7279) +- Zipper Duplicate Names (Issue [#80](https://github.com/IQSS/dataverse.harvard.edu/issues/80), PR #7276) +- Zipper Folders (Issue #7255, PR #7258) +- API for Solr cleanup (Issue #4225, PR #7211) ## Notes for Dataverse Installation Administrators -### xxx +### New Option for Dataset-level Large File Uploads + +- New API available for setting a dataset store ### New JVM Options and Database Settings From 628c2331a0b3de036a5c450feefaeec2d866fd40 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 10:59:15 -0400 Subject: [PATCH 050/113] use cases, some notes for admins --- doc/release-notes/5.1-release-notes.md | 28 +++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index 948756e4305..28e94779f9e 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -16,34 +16,30 @@ In previous releases, configuration options were added that allow each dataverse Newly-supported use cases in this release include: -- Multipart API upload (Issue #6763, PR #6995) +- Users can now upload files larger than 5 GB on installations running AWS S3 (Issue #6763, PR #6995) - Administrators will now be able to specify a store at the dataset level in addition to the Dataverse level (Issue #6872, PR #7272) -- Shapefiles bug (Issue #6873, PR #7279) -- Zipper Duplicate Names (Issue [#80](https://github.com/IQSS/dataverse.harvard.edu/issues/80), PR #7276) -- Zipper Folders (Issue #7255, PR #7258) -- API for Solr cleanup (Issue #4225, PR #7211) +- Users will have their dataset's directory structure retained when uploading a dataset with shapefiles (Issue #6873, PR #7279) +- Users will now be able to download zip files through the experimental Zipper service when the set of downloaded files have duplicate names (Issue [#80](https://github.com/IQSS/dataverse.harvard.edu/issues/80), PR #7276) +- Users will now be able to download zip files with the proper file structure through the experiment Zipper service (Issue #7255, PR #7258) +- Administrators will be able use a new API to keep the database and Solr index in sync (Issue #4225, PR #7211) ## Notes for Dataverse Installation Administrators -### New Option for Dataset-level Large File Uploads +### New API for setting a Dataset-level Store - New API available for setting a dataset store -### New JVM Options and Database Settings +### New API for keeping Solr records in sync -Several new JVM options and DB Settings have been added in this release. More documentation about each of these settings can be found in the Configuration section of the [Installation Guide](http://guides.dataverse.org/en/5.0/installation/config.rst). +From time to time, "stale" records may end up in Solr. This can affect the successful loading of a search results page on which the stale record exists. New API for Solr records -#### New JVM Options +### Documentation for Purging the Ingest Queue -- doi.dataciterestapiurlstring: Set with a value of "https://api.datacite.org" for production environments and "https://api.test.datacite.org" for test environments. Must be set if you are using DataCite as your DOI provider. - -#### New Database Settings - -- :CustomZipDownloadServiceUrl: If defined, this is the URL of the zipping service outside the main application server where zip downloads should be directed (instead of /api/access/datafiles/). +At times, it may be necessary to cancel long-running Ingest jobs in the interest of system stability. The Troubleshooting section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/) has specific steps. ## Notes for Tool Developers and Integrators -### xxx +### File Name change ## Complete List of Changes @@ -53,7 +49,7 @@ For help with upgrading, installing, or general questions please post to the [Da ## Installation -If this is a new installation, please see our [Installation Guide](http://guides.dataverse.org/en/5.0/installation/) +If this is a new installation, please see our [Installation Guide](http://guides.dataverse.org/en/5.1/installation/) ## Upgrade Instructions From fb80a54e557cb80736ca8a8c8fef1e5f26d7a2ea Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 11:03:07 -0400 Subject: [PATCH 051/113] deleting the old notes after integrating them into the main --- doc/release-notes/5.1-release-notes.md | 4 +++- doc/release-notes/6763-multipart-uploads.md | 3 --- doc/release-notes/6872-dataset-specific-stores | 8 -------- doc/release-notes/7184-spaces-in-filenames.md | 7 ------- 4 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 doc/release-notes/6763-multipart-uploads.md delete mode 100644 doc/release-notes/6872-dataset-specific-stores delete mode 100644 doc/release-notes/7184-spaces-in-filenames.md diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index 28e94779f9e..831eefaf89d 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -39,7 +39,9 @@ At times, it may be necessary to cancel long-running Ingest jobs in the interest ## Notes for Tool Developers and Integrators -### File Name change +### Spaces in File Names + +Dataverse Installations using S3 storage will no longer replace spaces in file names with the + character. If your tool or integration has any special handling around this, you may need to make further adjustments to maintain backwards compatibility while also supporting Dataverse installations on 5.1+. ## Complete List of Changes diff --git a/doc/release-notes/6763-multipart-uploads.md b/doc/release-notes/6763-multipart-uploads.md deleted file mode 100644 index ecec3efd9dc..00000000000 --- a/doc/release-notes/6763-multipart-uploads.md +++ /dev/null @@ -1,3 +0,0 @@ -# Large Data Support (continued) - -Direct S3 uploads now support multi-part uploading of large files (> 1GB by default) via the user interface and the API (which is used in the [Dataverse Uploader](https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader)). This allows uploads larger than 5 GB when using Amazon AWS S3 stores. \ No newline at end of file diff --git a/doc/release-notes/6872-dataset-specific-stores b/doc/release-notes/6872-dataset-specific-stores deleted file mode 100644 index f08aabe023f..00000000000 --- a/doc/release-notes/6872-dataset-specific-stores +++ /dev/null @@ -1,8 +0,0 @@ - -## Major Use Cases - -- Administrators will now be able to specify a store at the dataset level in addition to the Dataverse level (Issue #6872, PR #7272) - -## Notes for Administrators - -- New API available for setting a dataset store \ No newline at end of file diff --git a/doc/release-notes/7184-spaces-in-filenames.md b/doc/release-notes/7184-spaces-in-filenames.md deleted file mode 100644 index 1a5b41068ce..00000000000 --- a/doc/release-notes/7184-spaces-in-filenames.md +++ /dev/null @@ -1,7 +0,0 @@ -## Notes for Tool Developers and Integrators - -### Filenames - -Dataverse Installations using S3 storage will no longer replace spaces in file names with the + character. If your tool or integration has any special handling around this character change, you can remove it. - -(review this note if this is in the same release as the fix for #7188) \ No newline at end of file From d194201d37b9b6eda91a357a8092a3188c2a6483 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 11:42:09 -0400 Subject: [PATCH 052/113] installation steps update --- doc/release-notes/4225-stale-solr-records.md | 8 -- doc/release-notes/5.1-release-notes.md | 117 +++--------------- .../source/admin/solr-search-index.rst | 2 +- 3 files changed, 18 insertions(+), 109 deletions(-) delete mode 100644 doc/release-notes/4225-stale-solr-records.md diff --git a/doc/release-notes/4225-stale-solr-records.md b/doc/release-notes/4225-stale-solr-records.md deleted file mode 100644 index a2fe8c7c14a..00000000000 --- a/doc/release-notes/4225-stale-solr-records.md +++ /dev/null @@ -1,8 +0,0 @@ -## Major Use Cases - -- Administrators will be able to use new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause search results to not load. - -## Notes for Dataverse Installation Administrators - -New API endpoints...http://guides.dataverse.org/en/latest/admin/solr-search-index.html - diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index 831eefaf89d..e4b6d6bdea8 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -21,21 +21,21 @@ Newly-supported use cases in this release include: - Users will have their dataset's directory structure retained when uploading a dataset with shapefiles (Issue #6873, PR #7279) - Users will now be able to download zip files through the experimental Zipper service when the set of downloaded files have duplicate names (Issue [#80](https://github.com/IQSS/dataverse.harvard.edu/issues/80), PR #7276) - Users will now be able to download zip files with the proper file structure through the experiment Zipper service (Issue #7255, PR #7258) -- Administrators will be able use a new API to keep the database and Solr index in sync (Issue #4225, PR #7211) +- Administrators will be able to use new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause search results to not load. (Issue #4225, PR #7211) ## Notes for Dataverse Installation Administrators ### New API for setting a Dataset-level Store -- New API available for setting a dataset store +- This release adds a new API for setting a dataset-specific store. Learn more in the Managing Dataverse and Datasets section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/solr-search-index.html). -### New API for keeping Solr records in sync +### New APIs for keeping Solr records in sync -From time to time, "stale" records may end up in Solr. This can affect the successful loading of a search results page on which the stale record exists. New API for Solr records +This release adds new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause search results to not load. Learn more in the Solr section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/solr-search-index.html). ### Documentation for Purging the Ingest Queue -At times, it may be necessary to cancel long-running Ingest jobs in the interest of system stability. The Troubleshooting section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/) has specific steps. +At times, it may be necessary to cancel long-running Ingest jobs in the interest of system stability. The Troubleshooting section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/) now has specific steps. ## Notes for Tool Developers and Integrators @@ -57,105 +57,22 @@ If this is a new installation, please see our [Installation Guide](http://guides ### Upgrade from Glassfish 4.1 to Payara 5 -The instructions below describe the upgrade procedure based on moving an existing glassfish4 domain directory under Payara. We recommend this method instead of setting up a brand-new Payara domain using the installer because it appears to be the easiest way to recreate your current configuration and preserve all your data. +These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). -1. Download Payara, v5.2020.2 as of this writing: +### Upgrade from Dataverse 5.0 to Dataverse 5.1 - `curl -L -O https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip` - `sha256sum payara-5.2020.2.zip` - 1f5f7ea30901b1b4c7bcdfa5591881a700c9b7e2022ae3894192ba97eb83cc3e +1. Undeploy the previous version. -2. Unzip it somewhere (/usr/local is a safe bet) +/payara/bin/asadmin list-applications +/payara/bin/asadmin undeploy dataverse - `sudo unzip payara-5.2020.2.zip -d /usr/local/` +2. Stop payara and remove the generated directory, start. -3. Copy the Postgres driver to /usr/local/payara5/glassfish/lib +- service payara stop +- remove the generated directory: rm -rf payara/payara/domains/domain1/generated +- service payara start - `sudo cp /usr/local/glassfish4/glassfish/lib/postgresql-42.2.9.jar /usr/local/payara5/glassfish/lib/` +3. Deploy this version. +/payara/bin/asadmin deploy dataverse-5.1.war -4. Move payara5/glassfish/domains/domain1 out of the way - - `sudo mv /usr/local/payara5/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/domain1.orig` - -5. Undeploy the Dataverse web application (if deployed; version 4.20 is assumed in the example below) - - `sudo /usr/local/glassfish4/bin/asadmin list-applications` - `sudo /usr/local/glassfish4/bin/asadmin undeploy dataverse-4.20` - -6. Stop Glassfish; copy domain1 to Payara - - `sudo /usr/local/glassfish4/bin/asadmin stop-domain` - `sudo cp -ar /usr/local/glassfish4/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/` - -7. Remove the Glassfish cache directories - - `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` - -8. In domain.xml: - - Replace the -XX:PermSize and -XX:MaxPermSize JVM options with -XX:MetaspaceSize and -XX:MaxMetaspaceSize. - - -XX:MetaspaceSize=256m - -XX:MaxMetaspaceSize=512m - -Add the below JVM options beneath the -Ddataverse settings: - - -Dfish.payara.classloading.delegate=false - -XX:+UseG1GC - -XX:+UseStringDeduplication - -XX:+DisableExplicitGC - -9. Change any full pathnames /usr/local/glassfish4/... to /usr/local/payara5/... or whatever it is in your case. (Specifically check the -Ddataverse.files.directory and -Ddataverse.files.file.directory JVM options) - -10. In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above. - -(Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`: - - `sudo chown -R dataverse /usr/local/payara5/glassfish/domains/domain1` - `sudo chown -R dataverse /usr/local/payara5/glassfish/lib` - -11. You will also need to check that the service account has write permission on the files directory, if they are located outside the old Glassfish domain. And/or make sure the service account has the correct AWS credentials, if you are using S3 for storage. - -12. Finally, start Payara: - - `sudo -u dataverse /usr/local/payara5/bin/asadmin start-domain` - -13. Deploy the Dataverse 5 warfile: - - `sudo -u dataverse /usr/local/payara5/bin/asadmin deploy /path/to/dataverse-5.0.war` - -14. Then restart Payara: - - `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain` - `sudo -u dataverse /usr/local/payara5/bin/asadmin start-domain` - -### Additional Upgrade Steps - -1. Update Astrophysics Metadata Block (if used) - - `wget https://github.com/IQSS/dataverse/releases/download/5.0/astrophysics.tsv` - `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @astrophysics.tsv -H "Content-type: text/tab-separated-values"` - -2. (Recommended) Run ReExportall to update JSON Exports - - - -3. (Required for installations using DataCite) Add the JVM option doi.dataciterestapiurlstring - -For production environments: - - `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.datacite.org"` - -For test environments: - - `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"` - -The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example: - - `/usr/local/payara5/bin/asadmin delete-jvm-options "\-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"` - -4. (Recommended for installations using DataCite) Pre-register DOIs - -Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. - -Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. +4. Restart payara \ No newline at end of file diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index d37b7eedb26..bc5b3aae363 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -22,7 +22,7 @@ Get a list of all database objects that are missing in Solr, and Solr documents ``curl http://localhost:8080/api/admin/index/status`` -Remove all Solr documents that are orphaned (ie not associated with objects in the database): +Remove all Solr documents that are detached (ie not associated with objects in the database): ``curl http://localhost:8080/api/admin/index/clear-orphans`` From 2f7fb8ae0a2e3d5a05aecfd045d05ff5bb4749c2 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 12:08:43 -0400 Subject: [PATCH 053/113] code review feedback --- doc/release-notes/5.1-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index e4b6d6bdea8..d8f3d61eaa9 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -41,7 +41,7 @@ At times, it may be necessary to cancel long-running Ingest jobs in the interest ### Spaces in File Names -Dataverse Installations using S3 storage will no longer replace spaces in file names with the + character. If your tool or integration has any special handling around this, you may need to make further adjustments to maintain backwards compatibility while also supporting Dataverse installations on 5.1+. +Dataverse Installations using S3 storage will no longer replace spaces in file names of downloaded files with the + character. If your tool or integration has any special handling around this, you may need to make further adjustments to maintain backwards compatibility while also supporting Dataverse installations on 5.1+. ## Complete List of Changes From 3915ccb346374d34cae837c452e148b4142b41d4 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 13:59:28 -0400 Subject: [PATCH 054/113] Update doc/release-notes/5.1-release-notes.md Co-authored-by: Philip Durbin --- doc/release-notes/5.1-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index d8f3d61eaa9..6a48a2f2234 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -21,7 +21,7 @@ Newly-supported use cases in this release include: - Users will have their dataset's directory structure retained when uploading a dataset with shapefiles (Issue #6873, PR #7279) - Users will now be able to download zip files through the experimental Zipper service when the set of downloaded files have duplicate names (Issue [#80](https://github.com/IQSS/dataverse.harvard.edu/issues/80), PR #7276) - Users will now be able to download zip files with the proper file structure through the experiment Zipper service (Issue #7255, PR #7258) -- Administrators will be able to use new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause search results to not load. (Issue #4225, PR #7211) +- Administrators will be able to use new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause stale search results to not load. (Issue #4225, PR #7211) ## Notes for Dataverse Installation Administrators @@ -75,4 +75,4 @@ These instructions assume that you've already successfully upgraded from Dataver 3. Deploy this version. /payara/bin/asadmin deploy dataverse-5.1.war -4. Restart payara \ No newline at end of file +4. Restart payara From b86a9fc86c0881e333ad0d2e86211dcdf6a20b8c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 25 Sep 2020 14:00:18 -0400 Subject: [PATCH 055/113] Update 5.1-release-notes.md --- doc/release-notes/5.1-release-notes.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index d8f3d61eaa9..a62b807afbc 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -55,11 +55,7 @@ If this is a new installation, please see our [Installation Guide](http://guides ## Upgrade Instructions -### Upgrade from Glassfish 4.1 to Payara 5 - -These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). - -### Upgrade from Dataverse 5.0 to Dataverse 5.1 +0. These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). 1. Undeploy the previous version. From 06554b0e3cfcdddc3d7f7f151cd4f6ab32335174 Mon Sep 17 00:00:00 2001 From: jingma Date: Mon, 28 Sep 2020 22:37:00 +0200 Subject: [PATCH 056/113] Make identifier lower case. --- .../iq/dataverse/api/imports/ImportGenericServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index a91214b0dde..fd6ae849590 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -389,7 +389,7 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { if (!otherIds.isEmpty()) { // We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F" for (String otherId : otherIds) { - if (otherId.contains(GlobalId.DOI_PROTOCOL) || otherId.contains(GlobalId.HDL_PROTOCOL)) { + if (otherId.toLowerCase().contains(GlobalId.DOI_PROTOCOL) || otherId.toLowerCase().contains(GlobalId.HDL_PROTOCOL)) { return otherId; } } From 3d0524bac4f7fb44f0cc41c84b1fa3990e4ac331 Mon Sep 17 00:00:00 2001 From: jingma Date: Mon, 28 Sep 2020 22:44:27 +0200 Subject: [PATCH 057/113] Remove unused variable. --- .../iq/dataverse/api/imports/ImportGenericServiceBean.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 7cc6cac06a7..fd6ae849590 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -375,7 +375,6 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { if ("citation".equals(key)) { for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.otherId.equals(fieldDTO.getTypeName())) { - String otherId = ""; for (HashSet foo : fieldDTO.getMultipleCompound()) { for (FieldDTO next : foo) { if (DatasetFieldConstant.otherIdValue.equals(next.getTypeName())) { From e406037ae592dffebb0df96ffdf340fe0c43ba8f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 28 Sep 2020 19:15:05 -0400 Subject: [PATCH 058/113] The *correct* test file for the RestAssured shapefile processing test. (contains just one set of shape files, and no extra files). #6873 --- scripts/search/data/shape/shapefile.zip | Bin 57541 -> 57380 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/scripts/search/data/shape/shapefile.zip b/scripts/search/data/shape/shapefile.zip index 0bf5ee98c6cd0d1f3b1d08d68dbc307749662c16..c4da60f0b805aad6e8a13aea71447854893e08e1 100644 GIT binary patch delta 71 zcmX?lka@`g<_U&OhZ83nr!ehH+&Gh)2}o^T#Z13?0!#zNraV+ R0SH)uP=bkp;oDsh4*=PH7b^e& delta 213 zcmZ2-fcfY_<_U&OA0sCkr!bYVZk)-@1f(`EV_MDuqW!s7GJ@z|JpV5EGP7_ofWY?L zWdV}*-e)<1Y!K#QkYOm!&rK~!ttinesVE5z;bdSw`F&H|jmXUS(h6<{MwV}k3=AwH zK()a@bqXMbo4?(aVsd0;l4HhYs02_s0|QVa!`sGhcNx&EV1-yQ`Tu=!J~jq6hAbuq J2G@Ha9RNB%JFx%& From 20a1a08dc4259c20685fc952d76601754290ab3f Mon Sep 17 00:00:00 2001 From: jingma Date: Tue, 29 Sep 2020 02:19:51 +0200 Subject: [PATCH 059/113] Change to startsWith() and check for doi and hdl URLs. --- .../iq/dataverse/api/imports/ImportGenericServiceBean.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index fd6ae849590..d51407e7946 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -389,7 +389,8 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { if (!otherIds.isEmpty()) { // We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F" for (String otherId : otherIds) { - if (otherId.toLowerCase().contains(GlobalId.DOI_PROTOCOL) || otherId.toLowerCase().contains(GlobalId.HDL_PROTOCOL)) { + otherId = otherId.toLowerCase(); + if (otherId.startsWith(GlobalId.DOI_PROTOCOL) || otherId.startsWith(GlobalId.HDL_PROTOCOL) || otherId.startsWith(GlobalId.DOI_RESOLVER_URL) || otherId.startsWith(GlobalId.HDL_RESOLVER_URL)) { return otherId; } } From 1997b0ccb725a8b349a3272b11103ffde3193db6 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 29 Sep 2020 09:05:07 -0400 Subject: [PATCH 060/113] eliminates RunTimeException from DataCite registration (#7282) --- .../dataverse/DOIDataCiteRegisterService.java | 40 ++++++++------- .../iq/dataverse/DataCiteRESTfullClient.java | 50 +++++++------------ 2 files changed, 42 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index ca9e55e2f92..4ca391b896c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -118,20 +118,17 @@ public String reserveIdentifier(String identifier, Map metadata, } else { rc.setUrl(target); } - try { - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - } catch (UnsupportedEncodingException ex) { - Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - } - } else { - try { - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - } catch (UnsupportedEncodingException ex) { - Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - } } + + //try { + DataCiteRESTfullClient client = getClient(); + retString = client.postMetadata(xmlMetadata); + /* + See the comment about checking for this exception in the registerIdentifier() + method below + } catch (UnsupportedEncodingException ex) { + Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); + }*/ return retString; } @@ -149,21 +146,30 @@ public String registerIdentifier(String identifier, Map metadata } else { rc.setUrl(target); } - try { + //try { DataCiteRESTfullClient client = getClient(); retString = client.postMetadata(xmlMetadata); client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); + /* + Why are we specifically catching the UnsupportedEncoding exception here, + but not any others? Note that we are logging and ignoring the exception here... + so then the publicizeIdentifier() method in DOIDataCiteServiceBean + mistakenly assumes that the registration was a success! + } catch (UnsupportedEncodingException ex) { Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - } + }*/ } else { - try { + //try { DataCiteRESTfullClient client = getClient(); retString = client.postMetadata(xmlMetadata); client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); + /* + See the comment above, about checking for UnsupportedEncodingException ... + } catch (UnsupportedEncodingException ex) { Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - } + }*/ } return retString; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java index 913dc4d0034..491f19ab36c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java @@ -9,8 +9,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.UnsupportedEncodingException; - import java.util.logging.Level; import java.util.logging.Logger; @@ -102,24 +100,19 @@ public String getUrl(String doi) { * @param url * @return */ - public String postUrl(String doi, String url) throws UnsupportedEncodingException { + public String postUrl(String doi, String url) throws IOException { HttpPost httpPost = new HttpPost(this.url + "/doi"); httpPost.setHeader("Content-Type", "text/plain;charset=UTF-8"); httpPost.setEntity(new StringEntity("doi=" + doi + "\nurl=" + url, "utf-8")); - try { - HttpResponse response = httpClient.execute(httpPost,context); - String data = EntityUtils.toString(response.getEntity(), encoding); - if (response.getStatusLine().getStatusCode() != 201) { - String errMsg = "Response code: " + response.getStatusLine().getStatusCode() + ", " + data; - logger.log(Level.SEVERE,errMsg); - throw new RuntimeException(errMsg); - } - return data; - } catch (IOException ioe) { - logger.log(Level.SEVERE,"IOException when post url"); - throw new RuntimeException("IOException when post url", ioe); + HttpResponse response = httpClient.execute(httpPost, context); + String data = EntityUtils.toString(response.getEntity(), encoding); + if (response.getStatusLine().getStatusCode() != 201) { + String errMsg = "Response from postUrl: " + response.getStatusLine().getStatusCode() + ", " + data; + logger.log(Level.SEVERE, errMsg); + throw new IOException(errMsg); } + return data; } /** @@ -135,7 +128,7 @@ public String getMetadata(String doi) { HttpResponse response = httpClient.execute(httpGet,context); String data = EntityUtils.toString(response.getEntity(), encoding); if (response.getStatusLine().getStatusCode() != 200) { - String errMsg = "Response code: " + response.getStatusLine().getStatusCode() + ", " + data; + String errMsg = "Response from getMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; logger.log(Level.SEVERE, errMsg); throw new RuntimeException(errMsg); } @@ -152,21 +145,16 @@ public String getMetadata(String doi) { * @param doi * @return boolean true if identifier already exists on DataCite site */ - public boolean testDOIExists(String doi) { - HttpGet httpGet = new HttpGet(this.url + "/metadata/" + doi); - httpGet.setHeader("Accept", "application/xml"); - try { - HttpResponse response = httpClient.execute(httpGet,context); - if (response.getStatusLine().getStatusCode() != 200) { - EntityUtils.consumeQuietly(response.getEntity()); - return false; - } + public boolean testDOIExists(String doi) throws IOException { + HttpGet httpGet = new HttpGet(this.url + "/metadata/" + doi); + httpGet.setHeader("Accept", "application/xml"); + HttpResponse response = httpClient.execute(httpGet, context); + if (response.getStatusLine().getStatusCode() != 200) { EntityUtils.consumeQuietly(response.getEntity()); - return true; - } catch (IOException ioe) { - logger.log(Level.SEVERE, "IOException when get metadata"); - throw new RuntimeException("IOException when get metadata", ioe); - } + return false; + } + EntityUtils.consumeQuietly(response.getEntity()); + return true; } /** @@ -182,7 +170,7 @@ public String postMetadata(String metadata) throws IOException { HttpResponse response = httpClient.execute(httpPost, context); String data = EntityUtils.toString(response.getEntity(), encoding); if (response.getStatusLine().getStatusCode() != 201) { - String errMsg = "Response code: " + response.getStatusLine().getStatusCode() + ", " + data; + String errMsg = "Response from postMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; logger.log(Level.SEVERE, errMsg); throw new IOException(errMsg); } From 64310efec2cec227643b23ce757239bde87c038f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 29 Sep 2020 09:52:33 -0400 Subject: [PATCH 061/113] moving release note out of root --- 6359-release-notes.md => doc/release-notes/6359-release-notes.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 6359-release-notes.md => doc/release-notes/6359-release-notes.md (100%) diff --git a/6359-release-notes.md b/doc/release-notes/6359-release-notes.md similarity index 100% rename from 6359-release-notes.md rename to doc/release-notes/6359-release-notes.md From 23a02521685d1575fd7291030e84e86ee88fd8be Mon Sep 17 00:00:00 2001 From: jingma Date: Tue, 29 Sep 2020 19:10:57 +0200 Subject: [PATCH 062/113] Remove toLowerCase(). --- .../iq/dataverse/api/imports/ImportGenericServiceBean.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index d51407e7946..84195227b33 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -389,7 +389,6 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { if (!otherIds.isEmpty()) { // We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F" for (String otherId : otherIds) { - otherId = otherId.toLowerCase(); if (otherId.startsWith(GlobalId.DOI_PROTOCOL) || otherId.startsWith(GlobalId.HDL_PROTOCOL) || otherId.startsWith(GlobalId.DOI_RESOLVER_URL) || otherId.startsWith(GlobalId.HDL_RESOLVER_URL)) { return otherId; } From f0a0d0b9f31a8315a5043cc3f9f0a7b872af64a9 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 29 Sep 2020 13:41:07 -0400 Subject: [PATCH 063/113] updates from code review --- doc/release-notes/5.1-release-notes.md | 24 ++++++++++++++++++- doc/release-notes/6359-release-notes.md | 1 - .../source/admin/solr-search-index.rst | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) delete mode 100644 doc/release-notes/6359-release-notes.md diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index 41b841fc1bb..af0da488798 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -6,7 +6,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ### Large File Upload for Installations Using AWS S3 -The added support for multipart upload through the API (Issue #6763) will allow files larger than 5 GB to be uploaded to Dataverse when an installation is running on AWS S3. Previously, only non-AWS S3 storage configurations would allow uploads larger than 5 GB. +The added support for multipart upload through the API and UI (Issue #6763) will allow files larger than 5 GB to be uploaded to Dataverse when an installation is running on AWS S3. Previously, only non-AWS S3 storage configurations would allow uploads larger than 5 GB. ### Dataset-Specific Stores @@ -29,6 +29,12 @@ Newly-supported use cases in this release include: - This release adds a new API for setting a dataset-specific store. Learn more in the Managing Dataverse and Datasets section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/solr-search-index.html). +### Multipart Upload Storage Monitoring, Recommended Use for Multipart Upload + +Charges may be incurred for storage reserved for multipart uploads that are not completed or cancelled. Administrators may want to do periodic manual or automated checks for open multipart uploads. Learn more in the Big Data Support section of the [Developers Guide](http://guides.dataverse.org/en/5.1/developer/big-data-support.html). + +While multipart uploads can support much larger files, and can have advantages in terms of robust transfer and speed, they are more complex than single part direct uploads. Administrators should consider taking advantage of the options to limit use of multipart uploads to specific users by using multiple stores and configuring access to stores with high file size limits to specific Dataverses (added in 4.20) or Datasets (added in this release). + ### New APIs for keeping Solr records in sync This release adds new APIs to keep the Solr index and the DB in sync, allowing easier resolution of an issue that would occasionally cause search results to not load. Learn more in the Solr section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/solr-search-index.html). @@ -37,6 +43,10 @@ This release adds new APIs to keep the Solr index and the DB in sync, allowing e At times, it may be necessary to cancel long-running Ingest jobs in the interest of system stability. The Troubleshooting section of the [Admin Guide](http://guides.dataverse.org/en/5.1/admin/) now has specific steps. +### Biomedical Metadata Block Updated + +The Life Science Metadata block (biomedical.tsv) was updated. "Other Design Type", "Other Factor Type", "Other Technology Type", "Other Technology Platform" boxes were added. See the "Additional Upgrade Steps" below if you use this in your installation. + ## Notes for Tool Developers and Integrators ### Spaces in File Names @@ -72,3 +82,15 @@ If this is a new installation, please see our [Installation Guide](http://guides /payara/bin/asadmin deploy dataverse-5.1.war 4. Restart payara + +### Additional Upgrade Steps + +1. Update Biomedical Metadata Block (if used) + + `wget https://github.com/IQSS/dataverse/releases/download/5.1/biomedical.tsv` + `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` + +2. Updated Solr XML and Reload Solr + +- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory +- reload Solr, for example, http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1 diff --git a/doc/release-notes/6359-release-notes.md b/doc/release-notes/6359-release-notes.md deleted file mode 100644 index 6901b17ce60..00000000000 --- a/doc/release-notes/6359-release-notes.md +++ /dev/null @@ -1 +0,0 @@ -Life Science Metadata block (biomedical.tsv) was updated. "Other Design Type", "Other Factor Type", "Other Technology Type", "Other Technology Platform" boxes were added. To update the existing Life Science Metadata block run `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"`. To be able to publish one needs to reload solr: copy `schema_dv_mdb_fields.xml` and `schema_dv_mdb_copies.xml` to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory and reload the solr, for example, `http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1` diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index bc5b3aae363..d37b7eedb26 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -22,7 +22,7 @@ Get a list of all database objects that are missing in Solr, and Solr documents ``curl http://localhost:8080/api/admin/index/status`` -Remove all Solr documents that are detached (ie not associated with objects in the database): +Remove all Solr documents that are orphaned (ie not associated with objects in the database): ``curl http://localhost:8080/api/admin/index/clear-orphans`` From 727106b8a1eb238cfc83a2551fa5b9cc6d0446cc Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 29 Sep 2020 15:06:56 -0400 Subject: [PATCH 064/113] Adding JSON export re-run step --- doc/release-notes/5.1-release-notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index af0da488798..6fade7e78fc 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -94,3 +94,7 @@ If this is a new installation, please see our [Installation Guide](http://guides - copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory - reload Solr, for example, http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1 + +3. (Recommended) Run ReExportall to update JSON Exports + + From af09c9a5cf771731e3bc09c9504fcb4b94bf2c04 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 29 Sep 2020 16:30:34 -0400 Subject: [PATCH 065/113] code review updates --- doc/release-notes/5.1-release-notes.md | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index 6fade7e78fc..ac19dcbe50e 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -85,16 +85,11 @@ If this is a new installation, please see our [Installation Guide](http://guides ### Additional Upgrade Steps -1. Update Biomedical Metadata Block (if used) +1. Update Biomedical Metadata Block (if used), Reload Solr, ReExportAll `wget https://github.com/IQSS/dataverse/releases/download/5.1/biomedical.tsv` `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` - -2. Updated Solr XML and Reload Solr - -- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory -- reload Solr, for example, http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1 - -3. (Recommended) Run ReExportall to update JSON Exports - - +- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory +- reload Solr, for example, http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1 +- Run ReExportall to update JSON Exports + From 460d5d1e037589e4096f4fdb937802173ecc5c4a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 30 Sep 2020 12:18:34 -0400 Subject: [PATCH 066/113] A few verbose logging messages in the finalize publication command. It should not result in much noise, since publication is a relatively rare action. But it's important to know what's going on. (#7282) --- .../command/impl/FinalizeDatasetPublicationCommand.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 92bd22eb902..ce407d8986b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -66,6 +66,8 @@ public FinalizeDatasetPublicationCommand(Dataset aDataset, DataverseRequest aReq public Dataset execute(CommandContext ctxt) throws CommandException { Dataset theDataset = getDataset(); + logger.info("Finalizing publication of the dataset "+theDataset.getGlobalId().asString()); + // validate the physical files before we do anything else: // (unless specifically disabled; or a minor version) if (theDataset.getLatestVersion().getVersionState() != RELEASED @@ -91,6 +93,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { registerExternalIdentifier(theDataset, ctxt, false); } catch (CommandException comEx) { + logger.warning("Failed to reserve the identifier "+theDataset.getGlobalId().asString()+"; notifying the user(s), unlocking the dataset"); // Send failure notification to the user: notifyUsersDatasetPublishStatus(ctxt, theDataset, UserNotification.Type.PUBLISHFAILED_PIDREG); // Remove the dataset lock: @@ -197,6 +200,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.InReview); } + logger.info("Successfully published the dataset "+theDataset.getGlobalId().asString()); + + return readyDataset; } @@ -355,6 +361,8 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the idServiceBean. dataset.setIdentifierRegistered(true); } catch (Throwable e) { + logger.warning("Failed to register the identifier "+dataset.getGlobalId().asString()+", or to register a file in the dataset; notifying the user(s), unlocking the dataset"); + // Send failure notification to the user: notifyUsersDatasetPublishStatus(ctxt, dataset, UserNotification.Type.PUBLISHFAILED_PIDREG); From ad57432b35511dfd15536e9a4c1c02faf7e511e5 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 1 Oct 2020 12:27:11 -0400 Subject: [PATCH 067/113] removed some commented-out code and comments, that were left there for the code review. (#7282) --- .../dataverse/DOIDataCiteRegisterService.java | 43 ++++--------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 4ca391b896c..ba503a18d22 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -119,16 +119,10 @@ public String reserveIdentifier(String identifier, Map metadata, rc.setUrl(target); } } - - //try { - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - /* - See the comment about checking for this exception in the registerIdentifier() - method below - } catch (UnsupportedEncodingException ex) { - Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - }*/ + + DataCiteRESTfullClient client = getClient(); + retString = client.postMetadata(xmlMetadata); + return retString; } @@ -146,31 +140,12 @@ public String registerIdentifier(String identifier, Map metadata } else { rc.setUrl(target); } - //try { - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); - /* - Why are we specifically catching the UnsupportedEncoding exception here, - but not any others? Note that we are logging and ignoring the exception here... - so then the publicizeIdentifier() method in DOIDataCiteServiceBean - mistakenly assumes that the registration was a success! - - } catch (UnsupportedEncodingException ex) { - Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - }*/ - } else { - //try { - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); - /* - See the comment above, about checking for UnsupportedEncodingException ... - - } catch (UnsupportedEncodingException ex) { - Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); - }*/ } + + DataCiteRESTfullClient client = getClient(); + retString = client.postMetadata(xmlMetadata); + client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); + return retString; } From bf6ca00a8d1f4d0e1e941c3ac7bd40dcc29c7a59 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 10:26:17 -0400 Subject: [PATCH 068/113] add google archiver and dependencies --- pom.xml | 16 +- .../GoogleCloudSubmitToArchiveCommand.java | 228 ++++++++++++++++++ 2 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java diff --git a/pom.xml b/pom.xml index 6c9fa99dbc9..c86e1f0d608 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ - @@ -127,6 +127,13 @@ httpclient ${httpcomponents.client.version} + + com.google.cloud + google-cloud-bom + 0.115.0-alpha + pom + import + org.testcontainers testcontainers-bom @@ -137,7 +144,7 @@ @@ -581,6 +588,11 @@ opennlp-tools 1.9.1 + + com.google.cloud + google-cloud-storage + 1.97.0 + diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java new file mode 100644 index 00000000000..cb729a9807a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java @@ -0,0 +1,228 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; +import edu.harvard.iq.dataverse.DataCitation; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetLock.Reason; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.util.bagit.BagGenerator; +import edu.harvard.iq.dataverse.util.bagit.OREMap; +import edu.harvard.iq.dataverse.workflow.step.Failure; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.nio.charset.Charset; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; +import java.util.logging.Logger; + +import org.apache.commons.codec.binary.Hex; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +@RequiredPermissions(Permission.PublishDataset) +public class GoogleCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand implements Command { + + private static final Logger logger = Logger.getLogger(GoogleCloudSubmitToArchiveCommand.class.getName()); + private static final String GOOGLECLOUD_BUCKET = ":GoogleCloudBucket"; + private static final String GOOGLECLOUD_PROJECT = ":GoogleCloudProject"; + + public GoogleCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + super(aRequest, version); + } + + @Override + public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken token, Map requestedSettings) { + logger.fine("In GoogleCloudSubmitToArchiveCommand..."); + String bucketName = requestedSettings.get(GOOGLECLOUD_BUCKET); + String projectName = requestedSettings.get(GOOGLECLOUD_PROJECT); + logger.fine("Project: " + projectName + " Bucket: " + bucketName); + if (bucketName != null && projectName != null) { + Storage storage; + try { + FileInputStream fis = new FileInputStream(System.getProperty("dataverse.files.directory") + System.getProperty("file.separator")+ "googlecloudkey.json"); + storage = StorageOptions.newBuilder() + .setCredentials(ServiceAccountCredentials.fromStream(fis)) + .setProjectId(projectName) + .build() + .getService(); + Bucket bucket = storage.get(bucketName); + + Dataset dataset = dv.getDataset(); + if (dataset.getLockFor(Reason.finalizePublication) == null) { + + String spaceName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-') + .replace('.', '-').toLowerCase(); + + DataCitation dc = new DataCitation(dv); + Map metadata = dc.getDataCiteMetadata(); + String dataciteXml = DOIDataCiteRegisterService.getMetadataFromDvObject( + dv.getDataset().getGlobalId().asString(), metadata, dv.getDataset()); + String blobIdString = null; + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + try (PipedInputStream dataciteIn = new PipedInputStream(); DigestInputStream digestInputStream = new DigestInputStream(dataciteIn, messageDigest)) { + // Add datacite.xml file + + new Thread(new Runnable() { + public void run() { + try (PipedOutputStream dataciteOut = new PipedOutputStream(dataciteIn)) { + + dataciteOut.write(dataciteXml.getBytes(Charset.forName("utf-8"))); + dataciteOut.close(); + } catch (Exception e) { + logger.severe("Error creating datacite.xml: " + e.getMessage()); + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException("Error creating datacite.xml: " + e.getMessage()); + } + } + }).start(); + //Have seen broken pipe in PostPublishDataset workflow without this delay + int i=0; + while(digestInputStream.available()<=0 && i<100) { + Thread.sleep(10); + i++; + } + Blob dcXml = bucket.create(spaceName + "/datacite.v" + dv.getFriendlyVersionNumber()+".xml", digestInputStream, "text/xml", Bucket.BlobWriteOption.doesNotExist()); + String checksum = dcXml.getMd5ToHexString(); + logger.fine("Content: datacite.xml added with checksum: " + checksum); + String localchecksum = Hex.encodeHexString(digestInputStream.getMessageDigest().digest()); + if (!checksum.equals(localchecksum)) { + logger.severe(checksum + " not equal to " + localchecksum); + return new Failure("Error in transferring DataCite.xml file to GoogleCloud", + "GoogleCloud Submission Failure: incomplete metadata transfer"); + } + + // Store BagIt file + String fileName = spaceName + ".v" + dv.getFriendlyVersionNumber() + ".zip"; + + // Add BagIt ZIP file + // Google uses MD5 as one way to verify the + // transfer + messageDigest = MessageDigest.getInstance("MD5"); + try (PipedInputStream in = new PipedInputStream(100000); DigestInputStream digestInputStream2 = new DigestInputStream(in, messageDigest);) { + Thread writeThread = new Thread(new Runnable() { + public void run() { + try (PipedOutputStream out = new PipedOutputStream(in)) { + // Generate bag + BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); + bagger.setAuthenticationKey(token.getTokenString()); + bagger.generateBag(out); + } catch (Exception e) { + logger.severe("Error creating bag: " + e.getMessage()); + // TODO Auto-generated catch block + e.printStackTrace(); + try { + digestInputStream2.close(); + } catch(Exception ex) { + logger.warning(ex.getLocalizedMessage()); + } + throw new RuntimeException("Error creating bag: " + e.getMessage()); + } + } + }); + writeThread.start(); + /* + * The following loop handles two issues. First, with no delay, the + * bucket.create() call below can get started before the piped streams are set + * up, causing a failure (seen when triggered in a PostPublishDataset workflow). + * A minimal initial wait, e.g. until some bytes are available, would address + * this. Second, the BagGenerator class, due to it's use of parallel streaming + * creation of the zip file, has the characteristic that it makes a few bytes + * available - from setting up the directory structure for the zip file - + * significantly earlier than it is ready to stream file content (e.g. for + * thousands of files and GB of content). If, for these large datasets, + * bucket.create() is called as soon as bytes are available, the call can + * timeout before the bytes for all the zipped files are available. To manage + * this, the loop waits until 90K bytes are available, larger than any expected + * dir structure for the zip and implying that the main zipped content is + * available, or until the thread terminates, with all of its content written to + * the pipe. (Note the PipedInputStream buffer is set at 100K above - I didn't + * want to test whether that means that exactly 100K bytes will be available() + * for large datasets or not, so the test below is at 90K.) + * + * An additional sanity check limits the wait to 2K seconds. The BagGenerator + * has been used to archive >120K files, 2K directories, and ~600GB files on the + * SEAD project (streaming content to disk rather than over an internet + * connection) which would take longer than 2K seconds (10+ hours) and might + * produce an initial set of bytes for directories > 90K. If Dataverse ever + * needs to support datasets of this size, the numbers here would need to be + * increased, and/or a change in how archives are sent to google (e.g. as + * multiple blobs that get aggregated) would be required. + */ + i=0; + while(digestInputStream2.available()<=90000 && i<2000 && writeThread.isAlive()) { + Thread.sleep(1000); + logger.fine("avail: " + digestInputStream2.available() + " : " + writeThread.getState().toString()); + i++; + } + logger.fine("Bag: transfer started, i=" + i + ", avail = " + digestInputStream2.available()); + if(i==2000) { + throw new IOException("Stream not available"); + } + Blob bag = bucket.create(spaceName + "/" + fileName, digestInputStream2, "application/zip", Bucket.BlobWriteOption.doesNotExist()); + if(bag.getSize()==0) { + throw new IOException("Empty Bag"); + } + blobIdString = bag.getBlobId().getBucket() + "/" + bag.getBlobId().getName(); + checksum = bag.getMd5ToHexString(); + logger.fine("Bag: " + fileName + " added with checksum: " + checksum); + localchecksum = Hex.encodeHexString(digestInputStream2.getMessageDigest().digest()); + if (!checksum.equals(localchecksum)) { + logger.severe(checksum + " not equal to " + localchecksum); + return new Failure("Error in transferring Zip file to GoogleCloud", + "GoogleCloud Submission Failure: incomplete archive transfer"); + } + } catch (RuntimeException rte) { + logger.severe("Error creating Bag during GoogleCloud archiving: " + rte.getMessage()); + return new Failure("Error in generating Bag", + "GoogleCloud Submission Failure: archive file not created"); + } + + logger.fine("GoogleCloud Submission step: Content Transferred"); + + // Document the location of dataset archival copy location (actually the URL + // where you can + // view it as an admin) + + StringBuffer sb = new StringBuffer("https://console.cloud.google.com/storage/browser/"); + sb.append(blobIdString); + dv.setArchivalCopyLocation(sb.toString()); + } catch (RuntimeException rte) { + logger.severe("Error creating datacite xml file during GoogleCloud Archiving: " + rte.getMessage()); + return new Failure("Error in generating datacite.xml file", + "GoogleCloud Submission Failure: metadata file not created"); + } + } else { + logger.warning("GoogleCloud Submision Workflow aborted: Dataset locked for pidRegister"); + return new Failure("Dataset locked"); + } + } catch (Exception e) { + logger.warning(e.getLocalizedMessage()); + e.printStackTrace(); + return new Failure("GoogleCloud Submission Failure", + e.getLocalizedMessage() + ": check log for details"); + + } + return WorkflowStepResult.OK; + } else { + return new Failure("GoogleCloud Submission not configured - no \":GoogleCloudBucket\" and/or \":GoogleCloudProject\"."); + } + } + +} From ee08e9c2e2ec518248e5bb252ab7f7f1cf41876d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 13:32:57 -0400 Subject: [PATCH 069/113] documentation --- .../source/installation/config.rst | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 34da299528f..c9b3c681877 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -779,7 +779,7 @@ Dataverse may be configured to submit a copy of published Datasets, packaged as Dataverse offers an internal archive workflow which may be configured as a PostPublication workflow via an admin API call to manually submit previously published Datasets and prior versions to a configured archive such as Chronopolis. The workflow creates a `JSON-LD `_ serialized `OAI-ORE `_ map file, which is also available as a metadata export format in the Dataverse web interface. -At present, the DPNSubmitToArchiveCommand and LocalSubmitToArchiveCommand are the only implementations extending the AbstractSubmitToArchiveCommand and using the configurable mechanisms discussed below. +At present, the DPNSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchive are the only implementations extending the AbstractSubmitToArchiveCommand and using the configurable mechanisms discussed below. .. _Duracloud Configuration: @@ -831,6 +831,32 @@ ArchiverClassName - the fully qualified class to be used for archiving. For exam :BagItLocalPath is the file path that you've set in :ArchiverSettings. +.. _Google Cloud Configuration: + +Google Cloud Configuration +++++++++++++++++++++++++++ + +The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's could, including those in the 'Coldline' Storage class (cheaper, with slower access) + +``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.GoogleCloudSubmitToArchiveCommand"`` + +``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":":GoogleCloudBucket, :GoogleCloudProject"`` + +The Google Cloud archiver defines two custom settings, both are required: + +\:GoogleCloudBucket - the name of the bucket to use: + +``curl http://localhost:8080/api/admin/settings/:GoogleCloudBucket -X PUT -d "qdr-archive"`` + +\:GoogleCloudProject - the name of the project managing the bucket: + +``curl http://localhost:8080/api/admin/settings/:GoogleCloudProject -X PUT -d "qdr-project"`` + +In addition, the Google Cloud Archiver requires that the googlecloudkey.json file for the project be placed in the 'dataverse.files.directory' directory. This file can be created in the Google Could Console. + +.. _Local Path Configuration: + + API Call ++++++++ From 5dfbae394f875edf2ce8c724f9e3bcce45b8f7e0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 13:33:33 -0400 Subject: [PATCH 070/113] update DuraCloud archiver with enhancements from Google archiver --- .../impl/DuraCloudSubmitToArchiveCommand.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java index 66e8770a641..468e99f24c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java @@ -99,7 +99,12 @@ public void run() { } } }).start(); - + //Have seen Pipe Closed errors for other archivers when used as a workflow without this delay loop + int i=0; + while(digestInputStream.available()<=0 && i<100) { + Thread.sleep(10); + i++; + } String checksum = store.addContent(spaceName, "datacite.xml", digestInputStream, -1l, null, null, null); logger.fine("Content: datacite.xml added with checksum: " + checksum); @@ -133,7 +138,11 @@ public void run() { } } }).start(); - + i=0; + while(digestInputStream.available()<=0 && i<100) { + Thread.sleep(10); + i++; + } checksum = store.addContent(spaceName, fileName, digestInputStream2, -1l, null, null, null); logger.fine("Content: " + fileName + " added with checksum: " + checksum); @@ -174,6 +183,9 @@ public void run() { logger.severe(rte.getMessage()); return new Failure("Error in generating datacite.xml file", "DuraCloud Submission Failure: metadata file not created"); + } catch (InterruptedException e) { + logger.warning(e.getLocalizedMessage()); + e.printStackTrace(); } } catch (ContentStoreException e) { logger.warning(e.getMessage()); From fed5e456bd6381e3758d637cf69b5aec00641843 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 13:37:14 -0400 Subject: [PATCH 071/113] typos --- doc/sphinx-guides/source/installation/config.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index c9b3c681877..a9a532888aa 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -827,7 +827,7 @@ ArchiverClassName - the fully qualified class to be used for archiving. For exam \:ArchiverSettings - the archiver class can access required settings including existing Dataverse settings and dynamically defined ones specific to the class. This setting is a comma-separated list of those settings. For example\: -``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":BagItLocalPathâ€`` +``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":BagItLocalPath"`` :BagItLocalPath is the file path that you've set in :ArchiverSettings. @@ -836,7 +836,7 @@ ArchiverClassName - the fully qualified class to be used for archiving. For exam Google Cloud Configuration ++++++++++++++++++++++++++ -The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's could, including those in the 'Coldline' Storage class (cheaper, with slower access) +The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, including those in the 'Coldline' Storage class (cheaper, with slower access) ``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.GoogleCloudSubmitToArchiveCommand"`` @@ -852,7 +852,7 @@ The Google Cloud archiver defines two custom settings, both are required: ``curl http://localhost:8080/api/admin/settings/:GoogleCloudProject -X PUT -d "qdr-project"`` -In addition, the Google Cloud Archiver requires that the googlecloudkey.json file for the project be placed in the 'dataverse.files.directory' directory. This file can be created in the Google Could Console. +In addition, the Google Cloud Archiver requires that the googlecloudkey.json file for the project be placed in the 'dataverse.files.directory' directory. This file can be created in the Google Cloud Console. .. _Local Path Configuration: From 4d1b4b00bc90389947192340ff9a8a226a0b457e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 13:39:08 -0400 Subject: [PATCH 072/113] capitalization --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a9a532888aa..826f3472ab3 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -836,13 +836,13 @@ ArchiverClassName - the fully qualified class to be used for archiving. For exam Google Cloud Configuration ++++++++++++++++++++++++++ -The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, including those in the 'Coldline' Storage class (cheaper, with slower access) +The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, including those in the 'Coldline' storage class (cheaper, with slower access) ``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.GoogleCloudSubmitToArchiveCommand"`` ``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":":GoogleCloudBucket, :GoogleCloudProject"`` -The Google Cloud archiver defines two custom settings, both are required: +The Google Cloud Archiver defines two custom settings, both are required: \:GoogleCloudBucket - the name of the bucket to use: From 3eb0ebcab2facd9d79c40821f2903b3ae1c72155 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 13:40:08 -0400 Subject: [PATCH 073/113] for example --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 826f3472ab3..8aadeaf0601 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -844,11 +844,11 @@ The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, The Google Cloud Archiver defines two custom settings, both are required: -\:GoogleCloudBucket - the name of the bucket to use: +\:GoogleCloudBucket - the name of the bucket to use. For example: ``curl http://localhost:8080/api/admin/settings/:GoogleCloudBucket -X PUT -d "qdr-archive"`` -\:GoogleCloudProject - the name of the project managing the bucket: +\:GoogleCloudProject - the name of the project managing the bucket. For example: ``curl http://localhost:8080/api/admin/settings/:GoogleCloudProject -X PUT -d "qdr-project"`` From 3e5eecfae790dce775b194e5ca34ca0e61cbef50 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 15:47:05 -0400 Subject: [PATCH 074/113] adding settings to master list --- .../source/installation/config.rst | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8aadeaf0601..d44690a6eaf 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -772,6 +772,8 @@ For Google Analytics, the example script at :download:`analytics-code.html ` above for details about this and for further explanation of the other archiving related settings below). +This setting specifies which storage system to use by identifying the particular Java class that should be run. Current options include DuraCloudSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchiveCommand. + +``curl -X PUT -d 'LocalSubmitToArchiveCommand' http://localhost:8080/api/admin/settings/:ArchiverClassName`` + + +:ArchiverSettings + +Each Archiver class may have it's own custom settings. Along with setting which Archiver class to use, one must use this setting to identify which setting values should be sent to it when it is invoked. The value should be a comma-spearate list of setting names. +For example, the LocalSubmitToArchiveCommand only uses the :BagItLocalPath setting. To allow the class to use that setting, this setting must set as: + +``curl -X PUT -d ':BagItLocalPath' http://localhost:8080/api/admin/settings/:ArchiverSettings`` + +:DuraCloudHost +:DuraCloudPort +:DuraCloudContext + +These three settings define the host, port, and context used by the DuraCloudSubmitToArchiveCommand. :DuraCloudHost is required. The other settings have default values as noted in the :ref:`Duracloud Configuration ` section above. + +:BagItLocalPath + +This is the local file system path to be used with the LocalSubmitToArchiveCommand class. It is recommended to use an absolute path. See the :ref:`Local Path Configuration ` section above. + +:GoogleCloudBucket +:GoogleCloudProject + +These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration ` section above. \ No newline at end of file From ede2a221389839a4e6f70bcdacf6b4b58f45205e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 15:48:54 -0400 Subject: [PATCH 075/113] add formatting --- doc/sphinx-guides/source/installation/config.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d44690a6eaf..d58e23a3c99 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2157,8 +2157,8 @@ This setting specifies which storage system to use by identifying the particular ``curl -X PUT -d 'LocalSubmitToArchiveCommand' http://localhost:8080/api/admin/settings/:ArchiverClassName`` - :ArchiverSettings ++++++++++++++++++ Each Archiver class may have it's own custom settings. Along with setting which Archiver class to use, one must use this setting to identify which setting values should be sent to it when it is invoked. The value should be a comma-spearate list of setting names. For example, the LocalSubmitToArchiveCommand only uses the :BagItLocalPath setting. To allow the class to use that setting, this setting must set as: @@ -2166,16 +2166,22 @@ For example, the LocalSubmitToArchiveCommand only uses the :BagItLocalPath setti ``curl -X PUT -d ':BagItLocalPath' http://localhost:8080/api/admin/settings/:ArchiverSettings`` :DuraCloudHost +++++++++++++++ :DuraCloudPort +++++++++++++++ :DuraCloudContext ++++++++++++++++++ These three settings define the host, port, and context used by the DuraCloudSubmitToArchiveCommand. :DuraCloudHost is required. The other settings have default values as noted in the :ref:`Duracloud Configuration ` section above. :BagItLocalPath ++++++++++++++++ This is the local file system path to be used with the LocalSubmitToArchiveCommand class. It is recommended to use an absolute path. See the :ref:`Local Path Configuration ` section above. -:GoogleCloudBucket +:GoogleCloudBucket +++++++++++++++++++ :GoogleCloudProject ++++++++++++++++++++ These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration ` section above. \ No newline at end of file From a33b9b167a1c0b825966a55515388dcfbfdbd9dc Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 2 Oct 2020 15:54:52 -0400 Subject: [PATCH 076/113] simplify links --- doc/sphinx-guides/source/installation/config.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d58e23a3c99..a7268ad34ec 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2152,7 +2152,7 @@ To enable redirects to the zipper on a different server: :ArchiverClassName ++++++++++++++++++ -Dataverse can export archival "Bag' files to an extensible set of storage systems (see the :ref:`BagIt Export ` above for details about this and for further explanation of the other archiving related settings below). +Dataverse can export archival "Bag' files to an extensible set of storage systems (see :ref:`BagIt Export` above for details about this and for further explanation of the other archiving related settings below). This setting specifies which storage system to use by identifying the particular Java class that should be run. Current options include DuraCloudSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchiveCommand. ``curl -X PUT -d 'LocalSubmitToArchiveCommand' http://localhost:8080/api/admin/settings/:ArchiverClassName`` @@ -2172,16 +2172,16 @@ For example, the LocalSubmitToArchiveCommand only uses the :BagItLocalPath setti :DuraCloudContext +++++++++++++++++ -These three settings define the host, port, and context used by the DuraCloudSubmitToArchiveCommand. :DuraCloudHost is required. The other settings have default values as noted in the :ref:`Duracloud Configuration ` section above. +These three settings define the host, port, and context used by the DuraCloudSubmitToArchiveCommand. :DuraCloudHost is required. The other settings have default values as noted in the :ref:`Duracloud Configuration` section above. :BagItLocalPath +++++++++++++++ -This is the local file system path to be used with the LocalSubmitToArchiveCommand class. It is recommended to use an absolute path. See the :ref:`Local Path Configuration ` section above. +This is the local file system path to be used with the LocalSubmitToArchiveCommand class. It is recommended to use an absolute path. See the :ref:`Local Path Configuration` section above. :GoogleCloudBucket ++++++++++++++++++ :GoogleCloudProject +++++++++++++++++++ -These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration ` section above. \ No newline at end of file +These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. \ No newline at end of file From 4aeccfdf861cb3fb4e5710e8248a3ad5fe658ccb Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 2 Oct 2020 16:07:47 -0400 Subject: [PATCH 077/113] add release notes --- doc/release-notes/7140-google-cloud.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/release-notes/7140-google-cloud.md diff --git a/doc/release-notes/7140-google-cloud.md b/doc/release-notes/7140-google-cloud.md new file mode 100644 index 00000000000..62aef73acd0 --- /dev/null +++ b/doc/release-notes/7140-google-cloud.md @@ -0,0 +1,12 @@ +## Google Cloud Archiver + +Dataverse Bags can now be sent to a bucket in Google Cloud, including those in the 'Coldline' storage class, which provide less expensive but slower access. + +## Use Cases + +- As an Administrator I can set up a regular export to Google Cloud so that my users' data is preserved. + +## New Settings + +:GoogleCloudProject - the name of the project managing the bucket. +:GoogleCloudBucket - the name of the bucket to use \ No newline at end of file From efb4206ba430376d8bccb20e57dff8825f20966f Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 5 Oct 2020 12:56:34 -0400 Subject: [PATCH 078/113] Update "change dataset citation date" curl examples Changes the curl examples for "Set Citation Date Field Type for a Dataset" and "Revert Citation Date Field Type to Default for Dataset" endpoints to use the dataset's persistentId instead of its database ID. --- doc/sphinx-guides/source/api/native-api.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index bf778036f1b..fafe881ba01 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -989,16 +989,16 @@ Note that the dataset citation date field type must be a date field. export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - export ID=24 + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB export DATASET_FIELD_TYPE_NAME=:dateOfDeposit - curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/$ID/citationdate --data "$DATASET_FIELD_TYPE_NAME" + curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER --data "$DATASET_FIELD_TYPE_NAME" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/24/citationdate --data ":dateOfDeposit" + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB Revert Citation Date Field Type to Default for Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1009,15 +1009,15 @@ Restores the default citation date field type, ``:publicationDate``, for a given export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - export ID=24 + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB - curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID/citationdate + curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/citationdate + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB .. _list-roles-on-a-dataset-api: From c257a1ea5e2960459bf5bd9325994afee55f7a7c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 5 Oct 2020 13:29:09 -0400 Subject: [PATCH 079/113] Update doc/sphinx-guides/source/installation/config.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a7268ad34ec..414796aa7ca 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2160,7 +2160,7 @@ This setting specifies which storage system to use by identifying the particular :ArchiverSettings +++++++++++++++++ -Each Archiver class may have it's own custom settings. Along with setting which Archiver class to use, one must use this setting to identify which setting values should be sent to it when it is invoked. The value should be a comma-spearate list of setting names. +Each Archiver class may have its own custom settings. Along with setting which Archiver class to use, one must use this setting to identify which setting values should be sent to it when it is invoked. The value should be a comma-separated list of setting names. For example, the LocalSubmitToArchiveCommand only uses the :BagItLocalPath setting. To allow the class to use that setting, this setting must set as: ``curl -X PUT -d ':BagItLocalPath' http://localhost:8080/api/admin/settings/:ArchiverSettings`` @@ -2184,4 +2184,4 @@ This is the local file system path to be used with the LocalSubmitToArchiveComma :GoogleCloudProject +++++++++++++++++++ -These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. \ No newline at end of file +These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. From 57e6b426f48d1a0952694a451391d51fc8e85af6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 5 Oct 2020 13:29:28 -0400 Subject: [PATCH 080/113] Update doc/sphinx-guides/source/installation/config.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 414796aa7ca..a984bcfa0f4 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2152,7 +2152,7 @@ To enable redirects to the zipper on a different server: :ArchiverClassName ++++++++++++++++++ -Dataverse can export archival "Bag' files to an extensible set of storage systems (see :ref:`BagIt Export` above for details about this and for further explanation of the other archiving related settings below). +Dataverse can export archival "Bag" files to an extensible set of storage systems (see :ref:`BagIt Export` above for details about this and for further explanation of the other archiving related settings below). This setting specifies which storage system to use by identifying the particular Java class that should be run. Current options include DuraCloudSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchiveCommand. ``curl -X PUT -d 'LocalSubmitToArchiveCommand' http://localhost:8080/api/admin/settings/:ArchiverClassName`` From babb316d729817421c93421d8e5edda29e2dac90 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 5 Oct 2020 13:47:41 -0400 Subject: [PATCH 081/113] typo fixes --- doc/sphinx-guides/source/installation/config.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a7268ad34ec..a984bcfa0f4 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2152,7 +2152,7 @@ To enable redirects to the zipper on a different server: :ArchiverClassName ++++++++++++++++++ -Dataverse can export archival "Bag' files to an extensible set of storage systems (see :ref:`BagIt Export` above for details about this and for further explanation of the other archiving related settings below). +Dataverse can export archival "Bag" files to an extensible set of storage systems (see :ref:`BagIt Export` above for details about this and for further explanation of the other archiving related settings below). This setting specifies which storage system to use by identifying the particular Java class that should be run. Current options include DuraCloudSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchiveCommand. ``curl -X PUT -d 'LocalSubmitToArchiveCommand' http://localhost:8080/api/admin/settings/:ArchiverClassName`` @@ -2160,7 +2160,7 @@ This setting specifies which storage system to use by identifying the particular :ArchiverSettings +++++++++++++++++ -Each Archiver class may have it's own custom settings. Along with setting which Archiver class to use, one must use this setting to identify which setting values should be sent to it when it is invoked. The value should be a comma-spearate list of setting names. +Each Archiver class may have its own custom settings. Along with setting which Archiver class to use, one must use this setting to identify which setting values should be sent to it when it is invoked. The value should be a comma-separated list of setting names. For example, the LocalSubmitToArchiveCommand only uses the :BagItLocalPath setting. To allow the class to use that setting, this setting must set as: ``curl -X PUT -d ':BagItLocalPath' http://localhost:8080/api/admin/settings/:ArchiverSettings`` @@ -2184,4 +2184,4 @@ This is the local file system path to be used with the LocalSubmitToArchiveComma :GoogleCloudProject +++++++++++++++++++ -These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. \ No newline at end of file +These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. From 72b1ef1de1b2268626c452fab165b3592ff9a309 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 5 Oct 2020 15:13:18 -0400 Subject: [PATCH 082/113] Removed colon, corrected curl examples Removed colon from field name "dateOfDeposit", and completed the curl examples. --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fafe881ba01..3240ee9ebe0 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -990,7 +990,7 @@ Note that the dataset citation date field type must be a date field. export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB - export DATASET_FIELD_TYPE_NAME=:dateOfDeposit + export DATASET_FIELD_TYPE_NAME=dateOfDeposit curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER --data "$DATASET_FIELD_TYPE_NAME" @@ -998,7 +998,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB --data "dateOfDeposit" Revert Citation Date Field Type to Default for Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 023b1156ec859b7b6dc606e0c163b7946b5ca8ce Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Mon, 5 Oct 2020 15:51:45 -0400 Subject: [PATCH 083/113] Update conf.py Update to v5.1 --- doc/sphinx-guides/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 17c68d38468..2663328e264 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '5.0' +version = '5.1' # The full version, including alpha/beta/rc tags. -release = '5.0' +release = '5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From f8b7da3ecebdac7123ed04204ebf33ac9c397105 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Mon, 5 Oct 2020 15:52:31 -0400 Subject: [PATCH 084/113] Update versions.rst Update to v5.1 --- doc/sphinx-guides/source/versions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index a9f389fde54..dbd7ad0cf2c 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,8 +6,9 @@ Dataverse Documentation Versions This list provides a way to refer to the documentation for previous versions of Dataverse. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- 5.0 +- 5.1 +- `5.0 `__ - `4.20 `__ - `4.19 `__ - `4.18.1 `__ From 4b46b360f7ef63e9441b0f2bedf415cc3ef7baee Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Mon, 5 Oct 2020 15:53:33 -0400 Subject: [PATCH 085/113] Update pom.xml Update to v5.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c9fa99dbc9..a115e67fb19 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ --> edu.harvard.iq dataverse - 5.0 + 5.1 war dataverse From e1ad7f2371cb9e1a86cbf4ffdaae99ee79478952 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Mon, 5 Oct 2020 15:53:42 -0400 Subject: [PATCH 086/113] Update pom.xml Update to v5.1 From 700382110ddc1b22c0906e3a0eda0ec4f146861c Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Tue, 6 Oct 2020 10:51:37 -0400 Subject: [PATCH 087/113] #21 bump Apache Tika to 1.24.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a115e67fb19..e454b4c169b 100644 --- a/pom.xml +++ b/pom.xml @@ -573,7 +573,7 @@ org.apache.tika tika-parsers - 1.22 + 1.24.1 From 9a0769840165569e0b4c7d5f991d9cc495f4ba76 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Tue, 6 Oct 2020 13:32:00 -0400 Subject: [PATCH 088/113] #7298 correct broken biomedical.tsv link in upgrade instructions --- doc/release-notes/5.1-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index ac19dcbe50e..4a5b3943523 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -87,7 +87,7 @@ If this is a new installation, please see our [Installation Guide](http://guides 1. Update Biomedical Metadata Block (if used), Reload Solr, ReExportAll - `wget https://github.com/IQSS/dataverse/releases/download/5.1/biomedical.tsv` + `wget https://github.com/IQSS/dataverse/releases/download/v5.1/biomedical.tsv` `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` - copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory - reload Solr, for example, http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1 From 107d206d44dc63fb9f90b4ea97c693853c6c7372 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Tue, 6 Oct 2020 14:08:23 -0400 Subject: [PATCH 089/113] #7301 provide full Solr reload example --- doc/release-notes/5.1-release-notes.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/5.1-release-notes.md b/doc/release-notes/5.1-release-notes.md index ac19dcbe50e..8ec9e5746cf 100644 --- a/doc/release-notes/5.1-release-notes.md +++ b/doc/release-notes/5.1-release-notes.md @@ -89,7 +89,11 @@ If this is a new installation, please see our [Installation Guide](http://guides `wget https://github.com/IQSS/dataverse/releases/download/5.1/biomedical.tsv` `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` + - copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory -- reload Solr, for example, http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1 +- Restart Solr, or tell Solr to reload its configuration: + + `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"` + - Run ReExportall to update JSON Exports From c7738ec3f6080b5c31b2b92f7896ba9a3dde8e47 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 7 Oct 2020 10:00:50 -0400 Subject: [PATCH 090/113] up the pool size, don't repeatedly check that bucket exists --- .../iq/dataverse/dataaccess/S3AccessIO.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index c78b84233be..bbbe0383d0f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.dataaccess; import com.amazonaws.AmazonClientException; +import com.amazonaws.ClientConfiguration; import com.amazonaws.HttpMethod; import com.amazonaws.SdkClientException; import com.amazonaws.auth.profile.ProfileCredentialsProvider; @@ -88,7 +89,14 @@ public S3AccessIO(T dvObject, DataAccessRequest req, String driverId) { minPartSize = getMinPartSize(driverId); s3=getClient(driverId); tm=getTransferManager(driverId); - + //Not sure this is needed but moving it from the open method for now since it definitely doesn't need to run every time an object is opened. + try { + if (bucketName == null || !s3.doesBucketExistV2(bucketName)) { + throw new IOException("ERROR: S3AccessIO - You must create and configure a bucket before creating datasets."); + } + } catch (SdkClientException sce) { + throw new IOException("ERROR: S3AccessIO - Failed to look up bucket "+bucketName+" (is AWS properly configured?): " + sce.getMessage()); + } } catch (Exception e) { throw new AmazonClientException( "Cannot instantiate a S3 client; check your AWS credentials and region", @@ -124,14 +132,6 @@ public void open(DataAccessOption... options) throws IOException { throw new IOException("ERROR: s3 not initialised. "); } - try { - if (bucketName == null || !s3.doesBucketExist(bucketName)) { - throw new IOException("ERROR: S3AccessIO - You must create and configure a bucket before creating datasets."); - } - } catch (SdkClientException sce) { - throw new IOException("ERROR: S3AccessIO - Failed to look up bucket "+bucketName+" (is AWS properly configured?): " + sce.getMessage()); - } - DataAccessRequest req = this.getRequest(); if (isWriteAccessRequested(options)) { @@ -1043,6 +1043,10 @@ private static AmazonS3 getClient(String driverId) { // get a standard client, using the standard way of configuration the credentials, etc. AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard(); + ClientConfiguration cc = new ClientConfiguration(); + cc.setMaxConnections(4096); + s3CB.setClientConfiguration(cc); + /** * Pass in a URL pointing to your S3 compatible storage. * For possible values see https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/client/builder/AwsClientBuilder.EndpointConfiguration.html From 1fda2b5441cb46dfe0ec4872b80591b0efa878d9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 7 Oct 2020 11:38:10 -0400 Subject: [PATCH 091/113] configurable pool size with 256 default --- .../edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index bbbe0383d0f..a7c256192ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -1044,7 +1044,11 @@ private static AmazonS3 getClient(String driverId) { AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard(); ClientConfiguration cc = new ClientConfiguration(); - cc.setMaxConnections(4096); + Integer poolSize = Integer.getInteger("dataverse.files." + driverId + ".connection-pool-size"); + if(poolSize==null) { + poolSize = 256; + } + cc.setMaxConnections(poolSize); s3CB.setClientConfiguration(cc); /** From 0cbf2371885b2ba4e4db40adbd003d8ab5d57124 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 7 Oct 2020 12:15:17 -0400 Subject: [PATCH 092/113] doc update --- doc/sphinx-guides/source/installation/config.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 34da299528f..137144c8cdd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -516,6 +516,9 @@ By default, your store will use the [default] profile in you .aws configuration ``./asadmin create-jvm-options "-Ddataverse.files..profile="`` +Larger installations may want to increase the number of open S3 connections allowed (default is 256): For example, + +``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has @@ -541,6 +544,7 @@ dataverse.files..custom-endpoint-region Only used when dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` +dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` =========================================== ================== ========================================================================= ============= Reported Working S3-Compatible Storage From d6c39c49a025b787bd72a1680c7b5122acf39d7a Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 7 Oct 2020 12:22:20 -0400 Subject: [PATCH 093/113] add release note for new JVM option --- doc/release-notes/7308-connection-pool-size.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release-notes/7308-connection-pool-size.md diff --git a/doc/release-notes/7308-connection-pool-size.md b/doc/release-notes/7308-connection-pool-size.md new file mode 100644 index 00000000000..f0b32c56884 --- /dev/null +++ b/doc/release-notes/7308-connection-pool-size.md @@ -0,0 +1,9 @@ +### Notes to Installation Admins + +New JVM Option for connection pool size + +Larger installations may want to increase the number of open S3 connections allowed (default is 256): For example, + +``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"` + +(link to config guide) \ No newline at end of file From fd5172e1b18622cc62deb4bceac183718b9aaa29 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 7 Oct 2020 12:50:52 -0400 Subject: [PATCH 094/113] simplify per review --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index a7c256192ee..5de59524a55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -1044,10 +1044,7 @@ private static AmazonS3 getClient(String driverId) { AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard(); ClientConfiguration cc = new ClientConfiguration(); - Integer poolSize = Integer.getInteger("dataverse.files." + driverId + ".connection-pool-size"); - if(poolSize==null) { - poolSize = 256; - } + Integer poolSize = Integer.getInteger("dataverse.files." + driverId + ".connection-pool-size", 256); cc.setMaxConnections(poolSize); s3CB.setClientConfiguration(cc); From 00dc9b97b4a0d2df501b3300bc72f9899528c1f1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 7 Oct 2020 16:09:07 -0400 Subject: [PATCH 095/113] close any open streams/channels using try-with-resources where it looks straight forward, or IOUtils.closeQuietly() where it isn't. (The latter is deprecated in favor of using try-with-resources these days.) --- .../dataverse/dataaccess/DataConverter.java | 22 ++++++++---- .../dataaccess/ImageThumbConverter.java | 16 +++++---- .../iq/dataverse/dataaccess/S3AccessIO.java | 18 +++++----- .../dataaccess/StoredOriginalFile.java | 9 +++-- .../iq/dataverse/dataset/DatasetUtil.java | 9 ++++- .../command/impl/RedetectFileTypeCommand.java | 8 ++--- .../dataverse/ingest/IngestServiceBean.java | 36 +++++++++++-------- .../impl/plugins/dta/DTAFileReaderSpi.java | 19 +++++----- 8 files changed, 85 insertions(+), 52 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataConverter.java index a627df0dbb3..2c60b51a525 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataConverter.java @@ -26,6 +26,9 @@ import java.io.IOException; import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; + import java.util.List; import java.util.Map; import java.util.HashMap; @@ -171,7 +174,9 @@ public static File downloadFromStorageIO(StorageIO storageIO) { } else { try { storageIO.open(); - return downloadFromByteChannel(storageIO.getReadChannel(), storageIO.getSize()); + try (ReadableByteChannel tabFileChannel = storageIO.getReadChannel()) { + return downloadFromByteChannel(tabFileChannel, storageIO.getSize()); + } } catch (IOException ex) { logger.warning("caught IOException trying to store tabular file " + storageIO.getDataFile().getStorageIdentifier() + " as a temp file."); } @@ -184,12 +189,13 @@ private static File downloadFromByteChannel(ReadableByteChannel tabFileChannel, logger.fine("opening datafFileIO for the source tabular file..."); File tabFile = File.createTempFile("tempTabFile", ".tmp"); - FileChannel tempFileChannel = new FileOutputStream(tabFile).getChannel(); - tempFileChannel.transferFrom(tabFileChannel, 0, size); - return tabFile; + try (FileChannel tempFileChannel = new FileOutputStream(tabFile).getChannel();) { + tempFileChannel.transferFrom(tabFileChannel, 0, size); + return tabFile; + } } catch (IOException ioex) { logger.warning("caught IOException trying to store tabular file as a temp file."); - } + } return null; } @@ -237,8 +243,10 @@ private static File runFormatConversion (DataFile file, File tabFile, String for try { StorageIO storageIO = file.getStorageIO(); long size = storageIO.getAuxObjectSize("orig"); - File origFile = downloadFromByteChannel((ReadableByteChannel) storageIO.openAuxChannel("orig"), size); - resultInfo = dfs.directConvert(origFile, origFormat); + try (ReadableByteChannel origChannel = (ReadableByteChannel) storageIO.openAuxChannel("orig")) { + File origFile = downloadFromByteChannel(origChannel, size); + resultInfo = dfs.directConvert(origFile, origFormat); + } } catch (IOException ex) { ex.printStackTrace(); return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 1249e95494e..ec18f23a5a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -235,6 +235,7 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s return false; } finally { IOUtils.closeQuietly(tempFileChannel); + IOUtils.closeQuietly(pdfFileChannel); } sourcePdfFile = tempFile; } @@ -272,7 +273,9 @@ private static boolean generateImageThumbnail(StorageIO storageIO, int try { storageIO.open(); - return generateImageThumbnailFromInputStream(storageIO, size, storageIO.getInputStream()); + try(InputStream inputStream = storageIO.getInputStream()) { + return generateImageThumbnailFromInputStream(storageIO, size, inputStream); + } } catch (IOException ioex) { logger.warning("caught IOException trying to open an input stream for " + storageIO.getDataFile().getStorageIdentifier() + ioex); return false; @@ -312,6 +315,7 @@ private static boolean generateWorldMapThumbnail(StorageIO storageIO, worldMapImageInputStream.close(); return false; } + return generateImageThumbnailFromInputStream(storageIO, size, worldMapImageInputStream); } catch (FileNotFoundException fnfe) { logger.fine("No .img file for this worldmap file yet; giving up. Original Error: " + fnfe); return false; @@ -319,9 +323,10 @@ private static boolean generateWorldMapThumbnail(StorageIO storageIO, } catch (IOException ioex) { logger.warning("caught IOException trying to open an input stream for worldmap .img file (" + storageIO.getDataFile().getStorageIdentifier() + "). Original Error: " + ioex); return false; + } finally { + IOUtils.closeQuietly(worldMapImageInputStream); } - - return generateImageThumbnailFromInputStream(storageIO, size, worldMapImageInputStream); + } /* @@ -750,15 +755,14 @@ private static void rescaleImage(BufferedImage fullSizeImage, int width, int hei g2.drawImage(thumbImage, 0, 0, null); g2.dispose(); - try { - ImageOutputStream ios = ImageIO.createImageOutputStream(outputStream); + try (ImageOutputStream ios = ImageIO.createImageOutputStream(outputStream);) { + writer.setOutput(ios); // finally, save thumbnail image: writer.write(lowRes); writer.dispose(); - ios.close(); thumbImage.flush(); //fullSizeImage.flush(); lowRes.flush(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index c78b84233be..2d52d4ecbbe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -581,15 +581,17 @@ public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) thr private File createTempFile(Path path, InputStream inputStream) throws IOException { File targetFile = new File(path.toUri()); //File needs a name - OutputStream outStream = new FileOutputStream(targetFile); + try (OutputStream outStream = new FileOutputStream(targetFile);) { - byte[] buffer = new byte[8 * 1024]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead); - } - IOUtils.closeQuietly(inputStream); - IOUtils.closeQuietly(outStream); + byte[] buffer = new byte[8 * 1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outStream.write(buffer, 0, bytesRead); + } + + } finally { + IOUtils.closeQuietly(inputStream); + } return targetFile; } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StoredOriginalFile.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StoredOriginalFile.java index 5be7882bc5e..0955d5f5565 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StoredOriginalFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StoredOriginalFile.java @@ -26,6 +26,8 @@ import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.util.logging.Logger; + +import org.apache.tika.io.IOUtils; /** * * @author Leonid Andreev @@ -56,17 +58,18 @@ public static StorageIO retreive(StorageIO storageIO) { long storedOriginalSize; InputStreamIO inputStreamIO; - + Channel storedOriginalChannel = null; try { storageIO.open(); - Channel storedOriginalChannel = storageIO.openAuxChannel(SAVED_ORIGINAL_FILENAME_EXTENSION); + storedOriginalChannel = storageIO.openAuxChannel(SAVED_ORIGINAL_FILENAME_EXTENSION); storedOriginalSize = dataFile.getDataTable().getOriginalFileSize() != null ? dataFile.getDataTable().getOriginalFileSize() : storageIO.getAuxObjectSize(SAVED_ORIGINAL_FILENAME_EXTENSION); inputStreamIO = new InputStreamIO(Channels.newInputStream((ReadableByteChannel) storedOriginalChannel), storedOriginalSize); logger.fine("Opened stored original file as Aux "+SAVED_ORIGINAL_FILENAME_EXTENSION); } catch (IOException ioEx) { - // The original file not saved, or could not be opened. + IOUtils.closeQuietly(storedOriginalChannel); + // The original file not saved, or could not be opened. logger.fine("Failed to open stored original file as Aux "+SAVED_ORIGINAL_FILENAME_EXTENSION+"!"); return null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index fb8f151f4c1..69e35b8e8c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -273,7 +273,7 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat try { tmpFile = FileUtil.inputStreamToFile(inputStream); } catch (IOException ex) { - logger.severe(ex.getMessage()); + logger.severe(ex.getMessage()); } StorageIO dataAccess = null; @@ -298,11 +298,13 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat try { fullSizeImage = ImageIO.read(tmpFile); } catch (IOException ex) { + IOUtils.closeQuietly(inputStream); logger.severe(ex.getMessage()); return null; } if (fullSizeImage == null) { logger.fine("fullSizeImage was null!"); + IOUtils.closeQuietly(inputStream); return null; } int width = fullSizeImage.getWidth(); @@ -311,6 +313,7 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat try { src = new FileInputStream(tmpFile).getChannel(); } catch (FileNotFoundException ex) { + IOUtils.closeQuietly(inputStream); logger.severe(ex.getMessage()); return null; } @@ -318,6 +321,7 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat try { dest = new FileOutputStream(tmpFile).getChannel(); } catch (FileNotFoundException ex) { + IOUtils.closeQuietly(inputStream); logger.severe(ex.getMessage()); return null; } @@ -329,10 +333,13 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat } File tmpFileForResize = null; try { + //The stream was used around line 274 above, so this creates an empty file (OK since all it is used for is getting a path, but not reusing it here would make it easier to close it above.) tmpFileForResize = FileUtil.inputStreamToFile(inputStream); } catch (IOException ex) { logger.severe(ex.getMessage()); return null; + } finally { + IOUtils.closeQuietly(inputStream); } // We'll try to pre-generate the rescaled versions in both the // DEFAULT_DATASET_LOGO (currently 140) and DEFAULT_CARDIMAGE_SIZE (48) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java index 0477a483783..ef20ec76e12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RedetectFileTypeCommand.java @@ -53,11 +53,11 @@ public DataFile execute(CommandContext ctxt) throws CommandException { } else { // Need to create a temporary local file: - ReadableByteChannel targetFileChannel = (ReadableByteChannel) storageIO.getReadChannel(); tempFile = File.createTempFile("tempFileTypeCheck", ".tmp"); - FileChannel tempFileChannel = new FileOutputStream(tempFile).getChannel(); - tempFileChannel.transferFrom(targetFileChannel, 0, storageIO.getSize()); - + try (ReadableByteChannel targetFileChannel = (ReadableByteChannel) storageIO.getReadChannel(); + FileChannel tempFileChannel = new FileOutputStream(tempFile).getChannel();) { + tempFileChannel.transferFrom(targetFileChannel, 0, storageIO.getSize()); + } localFile = tempFile; } diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 81edeb28077..f5eeaa1c316 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -67,6 +67,8 @@ import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader; import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi; import edu.harvard.iq.dataverse.util.*; + +import org.apache.commons.io.IOUtils; //import edu.harvard.iq.dvn.unf.*; import org.dataverse.unf.*; import java.io.BufferedInputStream; @@ -823,12 +825,13 @@ public boolean ingestAsTabular(Long datafile_id) { localFile = storageIO.getFileSystemPath().toFile(); inputStream = new BufferedInputStream(storageIO.getInputStream()); } else { - ReadableByteChannel dataFileChannel = storageIO.getReadChannel(); + localFile = File.createTempFile("tempIngestSourceFile", ".tmp"); - FileChannel tempIngestSourceChannel = new FileOutputStream(localFile).getChannel(); + try (ReadableByteChannel dataFileChannel = storageIO.getReadChannel(); + FileChannel tempIngestSourceChannel = new FileOutputStream(localFile).getChannel();) { - tempIngestSourceChannel.transferFrom(dataFileChannel, 0, storageIO.getSize()); - + tempIngestSourceChannel.transferFrom(dataFileChannel, 0, storageIO.getSize()); + } inputStream = new BufferedInputStream(new FileInputStream(localFile)); logger.fine("Saved "+storageIO.getSize()+" bytes in a local temp file."); } @@ -895,6 +898,8 @@ public boolean ingestAsTabular(Long datafile_id) { logger.warning("Ingest failure (Exception " + unknownEx.getClass() + "): "+unknownEx.getMessage()+"."); return false; + } finally { + IOUtils.closeQuietly(inputStream); } String originalContentType = dataFile.getContentType(); @@ -1056,12 +1061,11 @@ private BufferedInputStream openFile(DataFile dataFile) throws IOException { if (storageIO.isLocalFile()) { inputStream = new BufferedInputStream(storageIO.getInputStream()); } else { - ReadableByteChannel dataFileChannel = storageIO.getReadChannel(); - File tempFile = File.createTempFile("tempIngestSourceFile", ".tmp"); - FileChannel tempIngestSourceChannel = new FileOutputStream(tempFile).getChannel(); - - tempIngestSourceChannel.transferFrom(dataFileChannel, 0, storageIO.getSize()); - + File tempFile = File.createTempFile("tempIngestSourceFile", ".tmp"); + try (ReadableByteChannel dataFileChannel = storageIO.getReadChannel(); + FileChannel tempIngestSourceChannel = new FileOutputStream(tempFile).getChannel();) { + tempIngestSourceChannel.transferFrom(dataFileChannel, 0, storageIO.getSize()); + } inputStream = new BufferedInputStream(new FileInputStream(tempFile)); logger.fine("Saved "+storageIO.getSize()+" bytes in a local temp file."); } @@ -1801,10 +1805,14 @@ private void fixMissingOriginalType(long fileId) { if (savedOriginalFile == null) { tempFileRequired = true; - ReadableByteChannel savedOriginalChannel = (ReadableByteChannel) storageIO.openAuxChannel(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); - savedOriginalFile = File.createTempFile("tempSavedOriginal", ".tmp"); - FileChannel tempSavedOriginalChannel = new FileOutputStream(savedOriginalFile).getChannel(); - tempSavedOriginalChannel.transferFrom(savedOriginalChannel, 0, storageIO.getAuxObjectSize(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION)); + savedOriginalFile = File.createTempFile("tempSavedOriginal", ".tmp"); + try (ReadableByteChannel savedOriginalChannel = (ReadableByteChannel) storageIO + .openAuxChannel(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); + FileChannel tempSavedOriginalChannel = new FileOutputStream(savedOriginalFile) + .getChannel();) { + tempSavedOriginalChannel.transferFrom(savedOriginalChannel, 0, + storageIO.getAuxObjectSize(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION)); + } } } catch (Exception ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReaderSpi.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReaderSpi.java index 519684d1668..53ce587fc75 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReaderSpi.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReaderSpi.java @@ -131,6 +131,7 @@ public boolean canDecodeInput(Object source) throws IOException { @Override public boolean canDecodeInput(BufferedInputStream stream) throws IOException { + //who closes this stream? if (stream ==null){ throw new IllegalArgumentException("stream == null!"); } @@ -185,21 +186,21 @@ public boolean canDecodeInput(File file) throws IOException { throw new IIOException("cannot read the input file"); } + byte[] hdr4 = new byte[4]; // set-up a FileChannel instance for a given file object - FileChannel srcChannel = new FileInputStream(file).getChannel(); - - // create a read-only MappedByteBuffer - MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, DTA_HEADER_SIZE); + try (FileChannel srcChannel = new FileInputStream(file).getChannel();) { - //printHexDump(buff, "hex dump of the byte-buffer"); + // create a read-only MappedByteBuffer + MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, DTA_HEADER_SIZE); - buff.rewind(); + // printHexDump(buff, "hex dump of the byte-buffer"); - dbgLog.fine("applying the dta test\n"); + buff.rewind(); - byte[] hdr4 = new byte[4]; - buff.get(hdr4, 0, 4); + dbgLog.fine("applying the dta test\n"); + buff.get(hdr4, 0, 4); + } dbgLog.fine("hex dump: 1st 4bytes =>" + new String(Hex.encodeHex(hdr4)) + "<-"); From ec791bd79cd52076ae9f80942c62eba59a9f5918 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 7 Oct 2020 16:14:01 -0400 Subject: [PATCH 096/113] formatting --- .../iq/dataverse/dataaccess/S3AccessIO.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 2d52d4ecbbe..0264ed7ffe1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -578,20 +578,20 @@ public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) thr //Helper method for supporting saving streams with unknown length to S3 //We save those streams to a file and then upload the file - private File createTempFile(Path path, InputStream inputStream) throws IOException { + private File createTempFile(Path path, InputStream inputStream) throws IOException { - File targetFile = new File(path.toUri()); //File needs a name - try (OutputStream outStream = new FileOutputStream(targetFile);) { + File targetFile = new File(path.toUri()); // File needs a name + try (OutputStream outStream = new FileOutputStream(targetFile);) { - byte[] buffer = new byte[8 * 1024]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead); - } + byte[] buffer = new byte[8 * 1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outStream.write(buffer, 0, bytesRead); + } - } finally { - IOUtils.closeQuietly(inputStream); - } + } finally { + IOUtils.closeQuietly(inputStream); + } return targetFile; } From b6db8c50e04ad477672ab754e4b56e80c77afec6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 7 Oct 2020 17:13:20 -0400 Subject: [PATCH 097/113] removing unused dependency --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index a115e67fb19..596a0d5ffae 100644 --- a/pom.xml +++ b/pom.xml @@ -440,11 +440,6 @@ slf4j-log4j12 1.6.1 - - axis - axis - 1.4 - io.searchbox jest From 09d282b0b0e4cb1f6237ee02a91481a43e424aad Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Thu, 8 Oct 2020 10:23:28 -0400 Subject: [PATCH 098/113] Update conf.py Update version to 5.1.1 --- doc/sphinx-guides/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 2663328e264..2cba6ba5491 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '5.1' +version = '5.1.1' # The full version, including alpha/beta/rc tags. -release = '5.1' +release = '5.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 998be8a8f1dbc61dd0367fa0f3388383846a60e6 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Thu, 8 Oct 2020 10:24:12 -0400 Subject: [PATCH 099/113] Update versions.rst Update to version 5.1.1 --- doc/sphinx-guides/source/versions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index dbd7ad0cf2c..0874d04f8ed 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,8 +6,9 @@ Dataverse Documentation Versions This list provides a way to refer to the documentation for previous versions of Dataverse. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- 5.1 +- 5.1.1 +- `5.1 `__ - `5.0 `__ - `4.20 `__ - `4.19 `__ From e7906501c3d127afc6970ea7a97580a7436d552f Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Thu, 8 Oct 2020 10:24:55 -0400 Subject: [PATCH 100/113] Update pom.xml Update to v5.1.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a115e67fb19..36756abdf4f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ --> edu.harvard.iq dataverse - 5.1 + 5.1.1 war dataverse From 01a59fc52e2efedab6be063725082957d840078f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 8 Oct 2020 13:12:16 -0400 Subject: [PATCH 101/113] Adding Release Notes --- doc/release-notes/5.1.1-release-notes.md | 59 +++++++++++++++++++ .../7308-connection-pool-size.md | 9 --- 2 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 doc/release-notes/5.1.1-release-notes.md delete mode 100644 doc/release-notes/7308-connection-pool-size.md diff --git a/doc/release-notes/5.1.1-release-notes.md b/doc/release-notes/5.1.1-release-notes.md new file mode 100644 index 00000000000..7b6c2a69a9d --- /dev/null +++ b/doc/release-notes/5.1.1-release-notes.md @@ -0,0 +1,59 @@ +# Dataverse 5.1.1 + +This minor release adds important optimizations to installations running on AWS S3. It is recommended that 5.1.1 be used in production instead of 5.1. + +## Release Highlights + +### Connection Pool Size Configuration Option, Connection Optimizations + +This release adds optimizations around closing streams and channels that may hold S3 http connections open and exhaust the connection pool. In parallel, this release adds the ability to increase the size of the connection pool, so a larger pool can be configured if needed. + +## Major Use Cases + +Newly-supported use cases in this release include: + +- Administrators of installations using S3 will be able to define the connection pool size, allowing better resource scaling for larger installations (Issue #7309, PR #7313) + +## Notes for Dataverse Installation Administrators + +### 5.1.1 vs. 5.1 for Production Use + +As mentioned above, we encourage 5.1.1 instead of 5.1 for production use. + +### New JVM Option for Connection Pool Size + +Larger installations may want to increase the number of open S3 connections allowed (default is 256). For example, to set the value to 4096: + +``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"` + +The JVM Options section of the [Configuration Guide](http://guides.dataverse.org/en/5.1.1/installation/config/) has more information. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [5.1.1 Milestone](https://github.com/IQSS/dataverse/milestone/91?closed=1) in Github. + +For help with upgrading, installing, or general questions please post to the [Dataverse Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please see our [Installation Guide](http://guides.dataverse.org/en/5.1.1/installation/) + +## Upgrade Instructions + +0. These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). + +1. Undeploy the previous version. + +/payara/bin/asadmin list-applications +/payara/bin/asadmin undeploy dataverse + +2. Stop payara and remove the generated directory, start. + +- service payara stop +- remove the generated directory: rm -rf payara/payara/domains/domain1/generated +- service payara start + +3. Deploy this version. +/payara/bin/asadmin deploy dataverse-5.1.1.war + +4. Restart payara diff --git a/doc/release-notes/7308-connection-pool-size.md b/doc/release-notes/7308-connection-pool-size.md deleted file mode 100644 index f0b32c56884..00000000000 --- a/doc/release-notes/7308-connection-pool-size.md +++ /dev/null @@ -1,9 +0,0 @@ -### Notes to Installation Admins - -New JVM Option for connection pool size - -Larger installations may want to increase the number of open S3 connections allowed (default is 256): For example, - -``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"` - -(link to config guide) \ No newline at end of file From d61b5933a2a8206695e72668d132a0a36c2f7a3e Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 8 Oct 2020 13:45:01 -0400 Subject: [PATCH 102/113] code review suggestions part 1 --- doc/release-notes/5.1.1-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.1.1-release-notes.md b/doc/release-notes/5.1.1-release-notes.md index 7b6c2a69a9d..c88e86dd882 100644 --- a/doc/release-notes/5.1.1-release-notes.md +++ b/doc/release-notes/5.1.1-release-notes.md @@ -6,7 +6,7 @@ This minor release adds important optimizations to installations running on AWS ### Connection Pool Size Configuration Option, Connection Optimizations -This release adds optimizations around closing streams and channels that may hold S3 http connections open and exhaust the connection pool. In parallel, this release adds the ability to increase the size of the connection pool, so a larger pool can be configured if needed. +Dataverse 5.1 improved the efficiency of making S3 connections through use of an http connection pool. This release adds optimizations around closing streams and channels that may hold S3 http connections open and exhaust the connection pool. In parallel, this release increases the default pool size from 50 to 256 and adds the ability to increase the size of the connection pool, so a larger pool can be configured if needed. ## Major Use Cases From 155dfb6030f4b8bd85ea17e9c8b98cc303038968 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 8 Oct 2020 13:47:31 -0400 Subject: [PATCH 103/113] code review feedback part 2 --- doc/release-notes/5.1.1-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.1.1-release-notes.md b/doc/release-notes/5.1.1-release-notes.md index c88e86dd882..f63870bc085 100644 --- a/doc/release-notes/5.1.1-release-notes.md +++ b/doc/release-notes/5.1.1-release-notes.md @@ -40,7 +40,7 @@ If this is a new installation, please see our [Installation Guide](http://guides ## Upgrade Instructions -0. These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). +0. These instructions assume that you've already successfully upgraded to Dataverse 5.1 following the instructions in the [Dataverse 5.1 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.1). 1. Undeploy the previous version. From 49af1973788b47aa53b68d0fe7c5741da4202c6c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 8 Oct 2020 13:48:32 -0400 Subject: [PATCH 104/113] code review feedback part 3 --- doc/release-notes/5.1.1-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.1.1-release-notes.md b/doc/release-notes/5.1.1-release-notes.md index f63870bc085..f5243aebc8f 100644 --- a/doc/release-notes/5.1.1-release-notes.md +++ b/doc/release-notes/5.1.1-release-notes.md @@ -1,6 +1,6 @@ # Dataverse 5.1.1 -This minor release adds important optimizations to installations running on AWS S3. It is recommended that 5.1.1 be used in production instead of 5.1. +This minor release adds important scaling improvements for installations running on AWS S3. It is recommended that 5.1.1 be used in production instead of 5.1. ## Release Highlights From ab29f15fa9b34b0806ad79f463a3f4339fed9d72 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 8 Oct 2020 15:27:29 -0400 Subject: [PATCH 105/113] typo --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 7733e8fc7d9..47cb2c797c9 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -846,7 +846,7 @@ The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, ``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.GoogleCloudSubmitToArchiveCommand"`` -``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":":GoogleCloudBucket, :GoogleCloudProject"`` +``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":GoogleCloudBucket, :GoogleCloudProject"`` The Google Cloud Archiver defines two custom settings, both are required: From af6dc98af51abe8b241dc00d4417e89ec4248e64 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 8 Oct 2020 16:22:10 -0400 Subject: [PATCH 106/113] expanded directions --- doc/sphinx-guides/source/installation/config.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 47cb2c797c9..677ac3f90f3 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -848,7 +848,9 @@ The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, ``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":GoogleCloudBucket, :GoogleCloudProject"`` -The Google Cloud Archiver defines two custom settings, both are required: +The Google Cloud Archiver defines two custom settings, both are required. The credentials for your account, in the form of a json key file, must also be obtained and stored locally (see below): + +In order to use the Google Cloud Archiver, you must have a Google account. You will need to create a project and bucket within that account and provide those values in the settings: \:GoogleCloudBucket - the name of the bucket to use. For example: @@ -858,7 +860,11 @@ The Google Cloud Archiver defines two custom settings, both are required: ``curl http://localhost:8080/api/admin/settings/:GoogleCloudProject -X PUT -d "qdr-project"`` -In addition, the Google Cloud Archiver requires that the googlecloudkey.json file for the project be placed in the 'dataverse.files.directory' directory. This file can be created in the Google Cloud Console. +The Google Cloud Archiver also requires a key file that must be renamed to 'googlecloudkey.json' file and placed in the directory identified by your 'dataverse.files.directory' jvm option. This file can be created in the Google Cloud Console. (One method: Navigate to your Project 'Settings'/'Service Accounts', create an account, give this account the 'Cloud Storage'/'Storage Admin' role, and once it's created, use the 'Actions' menu to 'Create Key', selecting the 'JSON' format option. Use this as the 'googlecloudkey.json' file.) + +For example: + +``cp /usr/local/payara5/glassfish/domains/domain1/files/googlecloudkey.json`` .. _Local Path Configuration: From 3484bb6b6a2cdd08a2e361d8d31de9dc56840939 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 8 Oct 2020 16:25:37 -0400 Subject: [PATCH 107/113] typos --- doc/sphinx-guides/source/installation/config.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 677ac3f90f3..de8fbad3687 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -860,14 +860,13 @@ In order to use the Google Cloud Archiver, you must have a Google account. You w ``curl http://localhost:8080/api/admin/settings/:GoogleCloudProject -X PUT -d "qdr-project"`` -The Google Cloud Archiver also requires a key file that must be renamed to 'googlecloudkey.json' file and placed in the directory identified by your 'dataverse.files.directory' jvm option. This file can be created in the Google Cloud Console. (One method: Navigate to your Project 'Settings'/'Service Accounts', create an account, give this account the 'Cloud Storage'/'Storage Admin' role, and once it's created, use the 'Actions' menu to 'Create Key', selecting the 'JSON' format option. Use this as the 'googlecloudkey.json' file.) +The Google Cloud Archiver also requires a key file that must be renamed to 'googlecloudkey.json' and placed in the directory identified by your 'dataverse.files.directory' jvm option. This file can be created in the Google Cloud Console. (One method: Navigate to your Project 'Settings'/'Service Accounts', create an account, give this account the 'Cloud Storage'/'Storage Admin' role, and once it's created, use the 'Actions' menu to 'Create Key', selecting the 'JSON' format option. Use this as the 'googlecloudkey.json' file.) For example: ``cp /usr/local/payara5/glassfish/domains/domain1/files/googlecloudkey.json`` -.. _Local Path Configuration: - +.. _Archiving API Call: API Call ++++++++ From 45172469e2f7042bc2183957b876452eacee0c04 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 9 Oct 2020 10:40:18 -0400 Subject: [PATCH 108/113] persist debug=true in session, toggle with query parameter #7273 --- .../harvard/iq/dataverse/DataverseSession.java | 18 +++++++++++++++++- .../search/SearchIncludeFragment.java | 12 +++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index e15badc994b..9e4bfb1b110 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -49,6 +49,14 @@ public class DataverseSession implements Serializable{ private static final Logger logger = Logger.getLogger(DataverseSession.class.getCanonicalName()); private boolean statusDismissed = false; + + /** + * If debug is set to true, some pages show extra debugging information. + * + * The way to set the boolean to true is to pass debug=true as a query + * parameter. The boolean will remain true until debug=false is passed. + */ + private boolean debug; public User getUser() { if ( user == null ) { @@ -82,7 +90,15 @@ public boolean isStatusDismissed() { public void setStatusDismissed(boolean status) { statusDismissed = status; //MAD: Set to true to enable code! } - + + public boolean isDebug() { + return debug; + } + + public void setDebug(boolean debug) { + this.debug = debug; + } + public StaticPermissionQuery on( Dataverse d ) { return permissionsService.userOn(user, d); } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index f5430ae32bb..464cdd05773 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -130,7 +130,7 @@ public class SearchIncludeFragment implements java.io.Serializable { Map staticSolrFieldFriendlyNamesBySolrField = new HashMap<>(); private boolean solrIsDown = false; private Map numberOfFacets = new HashMap<>(); - private boolean debug = false; + private Boolean debug; // private boolean showUnpublished; List filterQueriesDebug = new ArrayList<>(); // private Map friendlyName = new HashMap<>(); @@ -1017,13 +1017,15 @@ public void setRootDv(boolean rootDv) { this.rootDv = rootDv; } - public boolean isDebug() { - return (debug && session.getUser().isSuperuser()) + public Boolean getDebug() { + return (session.isDebug() && session.getUser().isSuperuser()) || settingsWrapper.isTrueForKey(":Debug", false); } - public void setDebug(boolean debug) { - this.debug = debug; + public void setDebug(Boolean debug) { + if (debug != null) { + session.setDebug(debug); + } } public List getFilterQueriesDebug() { From a4582f05e18a95f744c0db39b26afed4d1ed45bb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 14 Oct 2020 13:45:57 -0400 Subject: [PATCH 109/113] consolidate logic, remove :Debug database setting #7273 --- .../java/edu/harvard/iq/dataverse/DataverseSession.java | 7 ++++++- .../harvard/iq/dataverse/search/SearchIncludeFragment.java | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index 9e4bfb1b110..41c6db6f02d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -51,7 +51,8 @@ public class DataverseSession implements Serializable{ private boolean statusDismissed = false; /** - * If debug is set to true, some pages show extra debugging information. + * If debug is set to true, some pages show extra debugging information to + * superusers. * * The way to set the boolean to true is to pass debug=true as a query * parameter. The boolean will remain true until debug=false is passed. @@ -92,6 +93,10 @@ public void setStatusDismissed(boolean status) { } public boolean isDebug() { + // Only superusers get extra debugging information. + if (!getUser().isSuperuser()) { + return false; + } return debug; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 464cdd05773..541ab79c36d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1018,8 +1018,7 @@ public void setRootDv(boolean rootDv) { } public Boolean getDebug() { - return (session.isDebug() && session.getUser().isSuperuser()) - || settingsWrapper.isTrueForKey(":Debug", false); + return session.isDebug(); } public void setDebug(Boolean debug) { From ce78897f7c630dca5261f5b9bba42a9b3537db0e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 14 Oct 2020 16:09:32 -0400 Subject: [PATCH 110/113] switch to getters and setters in session bean #7273 --- .../iq/dataverse/DataverseSession.java | 20 +++++++++++++------ .../search/SearchIncludeFragment.java | 11 ---------- src/main/webapp/dataverse.xhtml | 2 +- src/main/webapp/search-include-fragment.xhtml | 10 +++++----- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index 41c6db6f02d..d8d73ceaf3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -54,10 +54,15 @@ public class DataverseSession implements Serializable{ * If debug is set to true, some pages show extra debugging information to * superusers. * - * The way to set the boolean to true is to pass debug=true as a query - * parameter. The boolean will remain true until debug=false is passed. + * The way to set the Boolean to true is to pass debug=true as a query + * parameter. The Boolean will remain true (even if nothing is passed to it) + * until debug=false is passed. + * + * Because a boolean is false by default when it comes from a viewParam we + * use a Boolean instead. That way, if the debug viewParam is null, we can + * leave the state alone (see setDebug()). */ - private boolean debug; + private Boolean debug; public User getUser() { if ( user == null ) { @@ -92,7 +97,7 @@ public void setStatusDismissed(boolean status) { statusDismissed = status; //MAD: Set to true to enable code! } - public boolean isDebug() { + public Boolean getDebug() { // Only superusers get extra debugging information. if (!getUser().isSuperuser()) { return false; @@ -100,8 +105,11 @@ public boolean isDebug() { return debug; } - public void setDebug(boolean debug) { - this.debug = debug; + public void setDebug(Boolean debug) { + // Leave the debug state alone if nothing is passed. + if (debug != null) { + this.debug = debug; + } } public StaticPermissionQuery on( Dataverse d ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 541ab79c36d..ac70072b7bb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -130,7 +130,6 @@ public class SearchIncludeFragment implements java.io.Serializable { Map staticSolrFieldFriendlyNamesBySolrField = new HashMap<>(); private boolean solrIsDown = false; private Map numberOfFacets = new HashMap<>(); - private Boolean debug; // private boolean showUnpublished; List filterQueriesDebug = new ArrayList<>(); // private Map friendlyName = new HashMap<>(); @@ -1017,16 +1016,6 @@ public void setRootDv(boolean rootDv) { this.rootDv = rootDv; } - public Boolean getDebug() { - return session.isDebug(); - } - - public void setDebug(Boolean debug) { - if (debug != null) { - session.setDebug(debug); - } - } - public List getFilterQueriesDebug() { return filterQueriesDebug; } diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 6d73b95b949..fbbe7563baf 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -44,7 +44,7 @@ - + diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index f3a4220bed6..3260495c4e7 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -232,7 +232,7 @@

    -
    +

    @@ -448,7 +448,7 @@ - + @@ -461,7 +461,7 @@ - + @@ -519,7 +519,7 @@ - + @@ -586,7 +586,7 @@ - + From af06ae3fa856490d6661a665e16f55803fecff9a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 14 Oct 2020 20:11:14 -0400 Subject: [PATCH 111/113] fix for handling a folder in a shapefile archive that's not explicitly listed in the zip directory. (#7331) --- .../edu/harvard/iq/dataverse/util/ShapefileHandler.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index d5e40dffaf3..3af562882f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -335,7 +335,14 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File String unzipFilePath = unzipFileName; if (unzipFolderName != null) { - unzipFilePath = unzipFolderName + "/" + unzipFileName; + unzipFilePath = unzipFolderName + "/" + unzipFileName; + + // There's a chance we haven't created this folder yet + // in the destination directory (this happens if the folder + // is not explicitly listed in the Zip archive directory). + String dirpath = target_directory.getAbsolutePath() + "/" + unzipFolderName; + // (and if it already exists, it'll be skipped) + createDirectory(dirpath); } if (unzipFileName==null){ From 049a2d2fb6a6cccf0515948983e4f5d786ab9c4b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 15 Oct 2020 12:13:22 -0400 Subject: [PATCH 112/113] giving an extra couple of seconds to the ingest in a test that's occasionally failing. --- src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java index 908beeac941..dde79574b87 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java @@ -350,7 +350,7 @@ public void downloadAllFilesTabular() throws IOException { .body("data.files[0].label", equalTo("50by1000.dta")); // UtilIT.MAXIMUM_INGEST_LOCK_DURATION is 3 but not long enough. - assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, 4)); + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION + 3)); Response downloadFiles1 = UtilIT.downloadFiles(datasetPid, apiToken); downloadFiles1.then().assertThat() From 0c4fccc45b542aac2173ca46551bd1fe6da1d31c Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 15 Oct 2020 17:22:39 -0400 Subject: [PATCH 113/113] A quick fix for the optimized datafile info retrieval method issue; this was as a result of the investigation into #7310 - ldjson export failure inside dataset page. --- .../edu/harvard/iq/dataverse/DataFileServiceBean.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 65d26d2eb63..a9a16e60ae2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -578,7 +578,7 @@ public void findFileMetadataOptimizedExperimental(Dataset owner, DatasetVersion int i = 0; - List dataTableResults = em.createNativeQuery("SELECT t0.ID, t0.DATAFILE_ID, t0.UNF, t0.CASEQUANTITY, t0.VARQUANTITY, t0.ORIGINALFILEFORMAT, t0.ORIGINALFILESIZE FROM dataTable t0, dataFile t1, dvObject t2 WHERE ((t0.DATAFILE_ID = t1.ID) AND (t1.ID = t2.ID) AND (t2.OWNER_ID = " + owner.getId() + ")) ORDER BY t0.ID").getResultList(); + List dataTableResults = em.createNativeQuery("SELECT t0.ID, t0.DATAFILE_ID, t0.UNF, t0.CASEQUANTITY, t0.VARQUANTITY, t0.ORIGINALFILEFORMAT, t0.ORIGINALFILESIZE, t0.ORIGINALFILENAME FROM dataTable t0, dataFile t1, dvObject t2 WHERE ((t0.DATAFILE_ID = t1.ID) AND (t1.ID = t2.ID) AND (t2.OWNER_ID = " + owner.getId() + ")) ORDER BY t0.ID").getResultList(); for (Object[] result : dataTableResults) { DataTable dataTable = new DataTable(); @@ -596,6 +596,8 @@ public void findFileMetadataOptimizedExperimental(Dataset owner, DatasetVersion dataTable.setOriginalFileSize((Long)result[6]); + dataTable.setOriginalFileName((String)result[7]); + dataTables.add(dataTable); datatableMap.put(fileId, i++); @@ -856,8 +858,10 @@ private List retrieveFileMetadataForVersion(Dataset dataset, Datas fileMetadata.setDatasetVersion(version); - //fileMetadata.setDataFile(dataset.getFiles().get(file_list_id)); + // Link the FileMetadata object to the DataFile: fileMetadata.setDataFile(dataFiles.get(file_list_id)); + // ... and the DataFile back to the FileMetadata: + fileMetadata.getDataFile().getFileMetadatas().add(fileMetadata); String description = (String) result[2];