Merge tag 'v5.5' into update_v5.5

TIK-NFL · Jun 10, 2021 · c646792 · c646792
2 parents 77410f9 + 5fc0150
commit c646792
Show file tree

Hide file tree

Showing 98 changed files with 4,857 additions and 2,481 deletions.
diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml
@@ -0,0 +1,14 @@
+name: "Guides Build Status"
+on: 
+  pull_request:
+    paths:
+      - 'doc/sphinx-guides/**/*.rst'
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ammaraskar/sphinx-action@master
+      with:
+        docs-folder: "doc/sphinx-guides/"
diff --git a/README.md b/README.md
@@ -21,6 +21,7 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
 [![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html)
 [![Unit Test Status](https://img.shields.io/travis/IQSS/dataverse?label=Unit%20Test%20Status)](https://travis-ci.org/IQSS/dataverse)
 [![Unit Test Coverage](https://img.shields.io/coveralls/github/IQSS/dataverse?label=Unit%20Test%20Coverage)](https://coveralls.io/github/IQSS/dataverse?branch=develop)
+[![Guides Build Status](https://github.com/IQSS/dataverse/actions/workflows/guides_build_sphinx.yml/badge.svg)](https://github.com/IQSS/dataverse/actions/workflows/guides_build_sphinx.yml)
 
 [dataverse.org]: https://dataverse.org
 [demo.dataverse.org]: https://demo.dataverse.org

diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile
@@ -2,9 +2,14 @@ FROM centos:8
 # OS dependencies
 # PG 10 is the default in centos8; keep the repo comment for when we bump to 11+
 #RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
-RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd
+
+RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf
+RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd python2 diffutils
 RUN yum install -y jq lsof awscli
 
+# for older search scripts
+RUN ln -s /usr/bin/python2 /usr/bin/python
+
 # copy and unpack dependencies (solr, payara)
 COPY dv /tmp/dv
 COPY testdata/schema*.xml /tmp/dv/

diff --git a/conf/docker-aio/configure_doi.bash b/conf/docker-aio/configure_doi.bash
@@ -1,24 +1,24 @@
 #!/usr/bin/env bash
 
-cd /usr/local/glassfish4
+cd /opt/payara5
 
 # if appropriate; reconfigure PID provider on the basis of environmental variables.
 if [ ! -z "${DoiProvider}" ]; then
         curl -X PUT -d ${DoiProvider} http://localhost:8080/api/admin/settings/:DoiProvider
 fi
 if [ ! -z "${doi_username}" ]; then
-        bin/asadmin create-jvm-options "-Ddoi.username=${doi_password}"
+        bin/asadmin create-jvm-options "-Ddoi.username=${doi_username}"
 fi
 if [ ! -z "${doi_password}" ]; then
         bin/asadmin create-jvm-options "-Ddoi.password=${doi_password}"
 fi
 if [ ! -z "${doi_baseurl}" ]; then
         bin/asadmin delete-jvm-options "-Ddoi.baseurlstring=https\://mds.test.datacite.org"
-        doi_baseurl_esc=`echo ${doi_baseurl} | sed -e 's/:/\\:/'`
-        bin/asadmin create-jvm-options "\"-Ddoi.baseurlstring=${doi_baseurl_esc}\""
+        doi_baseurl_esc=`echo ${doi_baseurl} | sed -e 's/:/\\\:/'`
+        bin/asadmin create-jvm-options "-Ddoi.baseurlstring=${doi_baseurl_esc}"
 fi
 if [ ! -z "${doi_dataciterestapiurl}" ]; then
         bin/asadmin delete-jvm-options "-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"
-        doi_dataciterestapiurl_esc=`echo ${doi_dataciterestapiurl} | sed -e 's/:/\\:/'`
-        bin/asadmin create-jvm-options "\"-Ddoi.dataciterestapiurlstring=${doi_dataciterestapiurl_esc}\""
+        doi_dataciterestapiurl_esc=`echo ${doi_dataciterestapiurl} | sed -e 's/:/\\\:/'`
+        bin/asadmin create-jvm-options "-Ddoi.dataciterestapiurlstring=${doi_dataciterestapiurl_esc}"
 fi
diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh
@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT -Ddataverse.test.baseurl=$dvurl
+source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT -Ddataverse.test.baseurl=$dvurl
diff --git a/conf/docker-aio/testscripts/install b/conf/docker-aio/testscripts/install
@@ -1,7 +1,7 @@
 #!/bin/sh
 export HOST_ADDRESS=localhost
-export GLASSFISH_ROOT=/usr/local/glassfish4
-export FILES_DIR=/usr/local/glassfish4/glassfish/domains/domain1/files
+export GLASSFISH_ROOT=/opt/payara5
+export FILES_DIR=/opt/payara5/glassfish/domains/domain1/files
 export DB_NAME=dvndb
 export DB_PORT=5432
 export DB_HOST=localhost

diff --git a/conf/docker-aio/testscripts/post b/conf/docker-aio/testscripts/post
@@ -2,7 +2,6 @@
 cd scripts/api
 ./setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out
 cd ../..
-psql -U dvnapp dvndb -f scripts/database/reference_data.sql
 psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/createsequence.sql
 scripts/search/tests/publish-dataverse-root
 #git checkout scripts/api/data/dv-root.json

diff --git a/conf/solr/8.8.1/schema.xml b/conf/solr/8.8.1/schema.xml
@@ -291,9 +291,166 @@
     <!-- <copyField source="*_i" dest="_text_" maxChars="3000"/> -->
 
     <!-- Dataverse copyField from http://localhost:8080/api/admin/index/solr/schema -->
-    <xi:include href="schema_dv_mdb_copies.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
-
-<!-- End: Dataverse-specific -->
+    <copyField source="accessToSources" dest="_text_" maxChars="3000"/>
+    <copyField source="actionsToMinimizeLoss" dest="_text_" maxChars="3000"/>
+    <copyField source="alternativeTitle" dest="_text_" maxChars="3000"/>
+    <copyField source="alternativeURL" dest="_text_" maxChars="3000"/>
+    <copyField source="astroFacility" dest="_text_" maxChars="3000"/>
+    <copyField source="astroInstrument" dest="_text_" maxChars="3000"/>
+    <copyField source="astroObject" dest="_text_" maxChars="3000"/>
+    <copyField source="astroType" dest="_text_" maxChars="3000"/>
+    <copyField source="author" dest="_text_" maxChars="3000"/>
+    <copyField source="authorAffiliation" dest="_text_" maxChars="3000"/>
+    <copyField source="authorIdentifier" dest="_text_" maxChars="3000"/>
+    <copyField source="authorIdentifierScheme" dest="_text_" maxChars="3000"/>
+    <copyField source="authorName" dest="_text_" maxChars="3000"/>
+    <copyField source="characteristicOfSources" dest="_text_" maxChars="3000"/>
+    <copyField source="city" dest="_text_" maxChars="3000"/>
+    <copyField source="cleaningOperations" dest="_text_" maxChars="3000"/>
+    <copyField source="collectionMode" dest="_text_" maxChars="3000"/>
+    <copyField source="collectorTraining" dest="_text_" maxChars="3000"/>
+    <copyField source="contributor" dest="_text_" maxChars="3000"/>
+    <copyField source="contributorName" dest="_text_" maxChars="3000"/>
+    <copyField source="contributorType" dest="_text_" maxChars="3000"/>
+    <copyField source="controlOperations" dest="_text_" maxChars="3000"/>
+    <copyField source="country" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Depth" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.ObjectCount" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.ObjectDensity" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Polarization" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Redshift.MaximumValue" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Redshift.MinimumValue" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.RedshiftValue" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.SkyFraction" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Spatial" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Spectral.Bandpass" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Spectral.CentralWavelength" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Spectral.MaximumWavelength" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Spectral.MinimumWavelength" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Spectral.Wavelength" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Temporal" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Temporal.StartTime" dest="_text_" maxChars="3000"/>
+    <copyField source="coverage.Temporal.StopTime" dest="_text_" maxChars="3000"/>
+    <copyField source="dataCollectionSituation" dest="_text_" maxChars="3000"/>
+    <copyField source="dataCollector" dest="_text_" maxChars="3000"/>
+    <copyField source="dataSources" dest="_text_" maxChars="3000"/>
+    <copyField source="datasetContact" dest="_text_" maxChars="3000"/>
+    <copyField source="datasetContactAffiliation" dest="_text_" maxChars="3000"/>
+    <copyField source="datasetContactEmail" dest="_text_" maxChars="3000"/>
+    <copyField source="datasetContactName" dest="_text_" maxChars="3000"/>
+    <copyField source="datasetLevelErrorNotes" dest="_text_" maxChars="3000"/>
+    <copyField source="dateOfCollection" dest="_text_" maxChars="3000"/>
+    <copyField source="dateOfCollectionEnd" dest="_text_" maxChars="3000"/>
+    <copyField source="dateOfCollectionStart" dest="_text_" maxChars="3000"/>
+    <copyField source="dateOfDeposit" dest="_text_" maxChars="3000"/>
+    <copyField source="depositor" dest="_text_" maxChars="3000"/>
+    <copyField source="deviationsFromSampleDesign" dest="_text_" maxChars="3000"/>
+    <copyField source="distributionDate" dest="_text_" maxChars="3000"/>
+    <copyField source="distributor" dest="_text_" maxChars="3000"/>
+    <copyField source="distributorAbbreviation" dest="_text_" maxChars="3000"/>
+    <copyField source="distributorAffiliation" dest="_text_" maxChars="3000"/>
+    <copyField source="distributorLogoURL" dest="_text_" maxChars="3000"/>
+    <copyField source="distributorName" dest="_text_" maxChars="3000"/>
+    <copyField source="distributorURL" dest="_text_" maxChars="3000"/>
+    <copyField source="dsDescription" dest="_text_" maxChars="3000"/>
+    <copyField source="dsDescriptionDate" dest="_text_" maxChars="3000"/>
+    <copyField source="dsDescriptionValue" dest="_text_" maxChars="3000"/>
+    <copyField source="eastLongitude" dest="_text_" maxChars="3000"/>
+    <copyField source="frequencyOfDataCollection" dest="_text_" maxChars="3000"/>
+    <copyField source="geographicBoundingBox" dest="_text_" maxChars="3000"/>
+    <copyField source="geographicCoverage" dest="_text_" maxChars="3000"/>
+    <copyField source="geographicUnit" dest="_text_" maxChars="3000"/>
+    <copyField source="grantNumber" dest="_text_" maxChars="3000"/>
+    <copyField source="grantNumberAgency" dest="_text_" maxChars="3000"/>
+    <copyField source="grantNumberValue" dest="_text_" maxChars="3000"/>
+    <copyField source="journalArticleType" dest="_text_" maxChars="3000"/>
+    <copyField source="journalIssue" dest="_text_" maxChars="3000"/>
+    <copyField source="journalPubDate" dest="_text_" maxChars="3000"/>
+    <copyField source="journalVolume" dest="_text_" maxChars="3000"/>
+    <copyField source="journalVolumeIssue" dest="_text_" maxChars="3000"/>
+    <copyField source="keyword" dest="_text_" maxChars="3000"/>
+    <copyField source="keywordValue" dest="_text_" maxChars="3000"/>
+    <copyField source="keywordVocabulary" dest="_text_" maxChars="3000"/>
+    <copyField source="keywordVocabularyURI" dest="_text_" maxChars="3000"/>
+    <copyField source="kindOfData" dest="_text_" maxChars="3000"/>
+    <copyField source="language" dest="_text_" maxChars="3000"/>
+    <copyField source="northLongitude" dest="_text_" maxChars="3000"/>
+    <copyField source="notesText" dest="_text_" maxChars="3000"/>
+    <copyField source="originOfSources" dest="_text_" maxChars="3000"/>
+    <copyField source="otherDataAppraisal" dest="_text_" maxChars="3000"/>
+    <copyField source="otherGeographicCoverage" dest="_text_" maxChars="3000"/>
+    <copyField source="otherId" dest="_text_" maxChars="3000"/>
+    <copyField source="otherIdAgency" dest="_text_" maxChars="3000"/>
+    <copyField source="otherIdValue" dest="_text_" maxChars="3000"/>
+    <copyField source="otherReferences" dest="_text_" maxChars="3000"/>
+    <copyField source="producer" dest="_text_" maxChars="3000"/>
+    <copyField source="producerAbbreviation" dest="_text_" maxChars="3000"/>
+    <copyField source="producerAffiliation" dest="_text_" maxChars="3000"/>
+    <copyField source="producerLogoURL" dest="_text_" maxChars="3000"/>
+    <copyField source="producerName" dest="_text_" maxChars="3000"/>
+    <copyField source="producerURL" dest="_text_" maxChars="3000"/>
+    <copyField source="productionDate" dest="_text_" maxChars="3000"/>
+    <copyField source="productionPlace" dest="_text_" maxChars="3000"/>
+    <copyField source="publication" dest="_text_" maxChars="3000"/>
+    <copyField source="publicationCitation" dest="_text_" maxChars="3000"/>
+    <copyField source="publicationIDNumber" dest="_text_" maxChars="3000"/>
+    <copyField source="publicationIDType" dest="_text_" maxChars="3000"/>
+    <copyField source="publicationURL" dest="_text_" maxChars="3000"/>
+    <copyField source="redshiftType" dest="_text_" maxChars="3000"/>
+    <copyField source="relatedDatasets" dest="_text_" maxChars="3000"/>
+    <copyField source="relatedMaterial" dest="_text_" maxChars="3000"/>
+    <copyField source="researchInstrument" dest="_text_" maxChars="3000"/>
+    <copyField source="resolution.Redshift" dest="_text_" maxChars="3000"/>
+    <copyField source="resolution.Spatial" dest="_text_" maxChars="3000"/>
+    <copyField source="resolution.Spectral" dest="_text_" maxChars="3000"/>
+    <copyField source="resolution.Temporal" dest="_text_" maxChars="3000"/>
+    <copyField source="responseRate" dest="_text_" maxChars="3000"/>
+    <copyField source="samplingErrorEstimates" dest="_text_" maxChars="3000"/>
+    <copyField source="samplingProcedure" dest="_text_" maxChars="3000"/>
+    <copyField source="series" dest="_text_" maxChars="3000"/>
+    <copyField source="seriesInformation" dest="_text_" maxChars="3000"/>
+    <copyField source="seriesName" dest="_text_" maxChars="3000"/>
+    <copyField source="socialScienceNotes" dest="_text_" maxChars="3000"/>
+    <copyField source="socialScienceNotesSubject" dest="_text_" maxChars="3000"/>
+    <copyField source="socialScienceNotesText" dest="_text_" maxChars="3000"/>
+    <copyField source="socialScienceNotesType" dest="_text_" maxChars="3000"/>
+    <copyField source="software" dest="_text_" maxChars="3000"/>
+    <copyField source="softwareName" dest="_text_" maxChars="3000"/>
+    <copyField source="softwareVersion" dest="_text_" maxChars="3000"/>
+    <copyField source="southLongitude" dest="_text_" maxChars="3000"/>
+    <copyField source="state" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayCellType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayMeasurementType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayOrganism" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayOtherMeasurmentType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayOtherOrganism" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayPlatform" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayOtherPlatform" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayTechnologyType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyAssayOtherTechnologyType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyDesignType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyOtherDesignType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyFactorType" dest="_text_" maxChars="3000"/>
+    <copyField source="studyOtherFactorType" dest="_text_" maxChars="3000"/>
+    <copyField source="subject" dest="_text_" maxChars="3000"/>
+    <copyField source="subtitle" dest="_text_" maxChars="3000"/>
+    <copyField source="targetSampleActualSize" dest="_text_" maxChars="3000"/>
+    <copyField source="targetSampleSize" dest="_text_" maxChars="3000"/>
+    <copyField source="targetSampleSizeFormula" dest="_text_" maxChars="3000"/>
+    <copyField source="timeMethod" dest="_text_" maxChars="3000"/>
+    <copyField source="timePeriodCovered" dest="_text_" maxChars="3000"/>
+    <copyField source="timePeriodCoveredEnd" dest="_text_" maxChars="3000"/>
+    <copyField source="timePeriodCoveredStart" dest="_text_" maxChars="3000"/>
+    <copyField source="title" dest="_text_" maxChars="3000"/>
+    <copyField source="topicClassValue" dest="_text_" maxChars="3000"/>
+    <copyField source="topicClassVocab" dest="_text_" maxChars="3000"/>
+    <copyField source="topicClassVocabURI" dest="_text_" maxChars="3000"/>
+    <copyField source="topicClassification" dest="_text_" maxChars="3000"/>
+    <copyField source="unitOfAnalysis" dest="_text_" maxChars="3000"/>
+    <copyField source="universe" dest="_text_" maxChars="3000"/>
+    <copyField source="weighting" dest="_text_" maxChars="3000"/>
+    <copyField source="westLongitude" dest="_text_" maxChars="3000"/>
+    <!-- End: Dataverse-specific -->
 
     <!-- This can be enabled, in case the client does not know what fields may be searched. It isn't enabled by default
          because it's very expensive to index everything twice. -->