Merge branch 'develop' into 10519-dataset-types #10519

Conflicts:
src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

.github/workflows/container_app_pr.yml

@@ -35,14 +35,14 @@ jobs:
 
             # Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets.
             - name: Login to Github Container Registry
-              uses: docker/login-action@v2
+              uses: docker/login-action@v3
               with:
                   registry: ghcr.io
                   username: ${{ secrets.GHCR_USERNAME }}
                   password: ${{ secrets.GHCR_TOKEN }}
 
             - name: Set up QEMU for multi-arch builds
-              uses: docker/setup-qemu-action@v2
+              uses: docker/setup-qemu-action@v3
 
             # Get the image tag from either the command or default to branch name (Not used for now)
             #-   name: Get the target tag name

.github/workflows/container_app_push.yml

@@ -126,20 +126,20 @@ jobs:
             # Depending on context, we push to different targets. Login accordingly.
             - if: github.event_name != 'pull_request'
               name: Log in to Docker Hub registry
-              uses: docker/login-action@v2
+              uses: docker/login-action@v3
               with:
                   username: ${{ secrets.DOCKERHUB_USERNAME }}
                   password: ${{ secrets.DOCKERHUB_TOKEN }}
             - if: ${{ github.event_name == 'pull_request' }}
               name: Login to Github Container Registry
-              uses: docker/login-action@v2
+              uses: docker/login-action@v3
               with:
                   registry: ghcr.io
                   username: ${{ secrets.GHCR_USERNAME }}
                   password: ${{ secrets.GHCR_TOKEN }}
 
             - name: Set up QEMU for multi-arch builds
-              uses: docker/setup-qemu-action@v2
+              uses: docker/setup-qemu-action@v3
 
             - name: Re-set image tag based on branch (if master)
               if: ${{ github.ref_name == 'master' }}

.github/workflows/deploy_beta_testing.yml

@@ -68,7 +68,7 @@ jobs:
           overwrite: true
 
       - name: Execute payara war deployment remotely
-        uses: appleboy/ssh-action@v1.0.0
+        uses: appleboy/ssh-action@v1.2.0
         env:
           INPUT_WAR_FILE: ${{ env.war_file }}
         with:

conf/solr/schema.xml

@@ -254,6 +254,21 @@
         WARNING: Do not remove the following include guards if you intend to use the neat helper scripts we provide.
     -->
     <!-- SCHEMA-FIELDS::BEGIN -->
+    <field name="3d3DTechnique" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dAltText" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dEquipment" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dExportedFileFormat" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dExportedFilePolygonCount" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dHandling" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dHeight" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dLength" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dLightingSetup" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dMasterFilePolygonCount" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dMaterialComposition" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dObjectDimensions" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dUnit" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dWeight" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dWidth" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="accessToSources" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="actionsToMinimizeLoss" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="alternativeTitle" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -496,6 +511,21 @@
         WARNING: Do not remove the following include guards if you intend to use the neat helper scripts we provide.
     -->
     <!-- SCHEMA-COPY-FIELDS::BEGIN -->
+    <copyField source="3d3DTechnique" dest="_text_" maxChars="3000"/>
+    <copyField source="3dAltText" dest="_text_" maxChars="3000"/>
+    <copyField source="3dEquipment" dest="_text_" maxChars="3000"/>
+    <copyField source="3dExportedFileFormat" dest="_text_" maxChars="3000"/>
+    <copyField source="3dExportedFilePolygonCount" dest="_text_" maxChars="3000"/>
+    <copyField source="3dHandling" dest="_text_" maxChars="3000"/>
+    <copyField source="3dHeight" dest="_text_" maxChars="3000"/>
+    <copyField source="3dLength" dest="_text_" maxChars="3000"/>
+    <copyField source="3dLightingSetup" dest="_text_" maxChars="3000"/>
+    <copyField source="3dMasterFilePolygonCount" dest="_text_" maxChars="3000"/>
+    <copyField source="3dMaterialComposition" dest="_text_" maxChars="3000"/>
+    <copyField source="3dObjectDimensions" dest="_text_" maxChars="3000"/>
+    <copyField source="3dUnit" dest="_text_" maxChars="3000"/>
+    <copyField source="3dWeight" dest="_text_" maxChars="3000"/>
+    <copyField source="3dWidth" dest="_text_" maxChars="3000"/>
     <copyField source="accessToSources" dest="_text_" maxChars="3000"/>
     <copyField source="actionsToMinimizeLoss" dest="_text_" maxChars="3000"/>
     <copyField source="alternativeTitle" dest="_text_" maxChars="3000"/>

doc/release-notes/10887-solr-field-types.md

@@ -73,6 +73,26 @@ Now start Solr.
 
 8\. Reindex Solr
 
+Note: these instructions are a little different than usual because we observed a strange error about `DOCS_AND_FREQS_AND_POSITIONS` when testing upgrades (see #11139 for details). Extra steps about explicitly clearing the index and reloading the core are included. If you run into trouble, as a last resort, you could reinstall Solr completely and then reindex.
+
+Clear the Solr index:
+
+```shell
+curl http://localhost:8080/api/admin/index/clear
+```
+
+Make sure the Solr index is empty:
+
+```shell
+curl "http://localhost:8983/solr/collection1/select?rows=1000000&wt=json&indent=true&q=*%3A*"
+```
+
+Reload the Solr core:
+
+```shell
+curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"
+```
+
 Below is the simplest way to reindex Solr:
 
 ```shell

doc/release-notes/10943-featured-items.md

@@ -0,0 +1,15 @@
+CRUD endpoints for Collection Featured Items have been implemented. In particular, the following endpoints have been implemented:
+
+- Create a feature item (POST /api/dataverses/<dataverse_id>/featuredItems)
+- Update a feature item (PUT /api/dataverseFeaturedItems/<item_id>)
+- Delete a feature item (DELETE /api/dataverseFeaturedItems/<item_id>)
+- List all featured items in a collection (GET /api/dataverses/<dataverse_id>/featuredItems)
+- Delete all featured items in a collection (DELETE /api/dataverses/<dataverse_id>/featuredItems)
+- Update all featured items in a collection (PUT /api/dataverses/<dataverse_id>/featuredItems)
+
+New settings:
+
+- dataverse.files.featured-items.image-maxsize - It sets the maximum allowed size of the image that can be added to a featured item.
+- dataverse.files.featured-items.image-uploads - It specifies the name of the subdirectory for saving featured item images within the docroot directory.
+
+See also #10943 and #11124.

doc/release-notes/11120-add-3d-object-metadata-block.md

@@ -0,0 +1,60 @@
+### New 3D Object Data Metadata Block
+
+A new metadata block has been added for describing 3D object data. You can download it from the [guides](https://dataverse-guide--11167.org.readthedocs.build/en/11167/user/appendix.html). See also #11120 and #11167.
+
+All new Dataverse installations will receive this metadata block by default. We recommend adding it by following the upgrade instructions below.
+
+## Upgrade Instructions
+
+### For 6.6-Release-notes.md
+
+6\. Restart Payara
+
+7\. Update metadata blocks
+
+These changes reflect incremental improvements made to the handling of core metadata fields.
+
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/citation.tsv
+
+curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv
+```
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/3d_objects.tsv
+
+curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file 3d_objects.tsv
+```
+
+8\. Update Solr schema.xml file. Start with the standard v6.6 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 8a).
+
+Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.6/installation/prerequisites.html#solr-init-script)).
+
+```shell
+service solr stop
+```
+
+Replace schema.xml
+
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/schema.xml
+cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf
+```
+
+Start Solr (but if you use any custom metadata blocks or adding 3D Objects, perform the next step, 8a first).
+
+```shell
+service solr start
+```
+8a\. For installations with custom or experimental metadata blocks:
+
+Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different):
+
+```shell
+	wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/update-fields.sh
+	chmod +x update-fields.sh
+	curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml
+```
+
+Now start Solr.
+
+9\. Reindex Solr

doc/release-notes/11127-update-search-api-to-show-dvobject-type-counts.md

@@ -0,0 +1,5 @@
+### show-type-counts Behavior changed in Search API
+
+In the Search API if you set show_type_counts=true the response will include all object types (Dataverses, Datasets, and Files) even if the search result for any given type is 0.
+
+See also the [guides](https://preview.guides.gdcc.io/en/develop/api/search.html#parameters), #11127 and #11138.

doc/release-notes/11129-send-feedback-to-contacts.md

@@ -0,0 +1,14 @@
+This feature adds a new API to send feedback to the Collection, Dataset, or DataFile's contacts.
+Similar to the "admin/feedback" API the "sendfeedback" API sends an email to all the contacts listed for the Dataset. The main differences for this feature are:
+1. This API is not limited to Admins
+2. This API does not return the email addresses in the "toEmail" and "ccEmail" elements for privacy reasons
+3. This API can be rate limited to avoid spamming
+4. The body size limit can be configured
+5. The body will be stripped of any html code to prevent malicious scripts or links
+6. The fromEmail will be validated for correct format
+
+To set the Rate Limiting for guest users (See Rate Limiting Configuration for more details. This example allows 1 send per hour for any guest)
+``curl http://localhost:8080/api/admin/settings/:RateLimitingCapacityByTierAndAction -X PUT -d '[{\"tier\": 0, \"limitPerHour\": 1, \"actions\": [\"CheckRateLimitForDatasetFeedbackCommand\"]}]'``
+
+To set the message size limit (example limit of 1080 chars):
+``curl -X PUT -d 1080 http://localhost:8080/api/admin/settings/:ContactFeedbackMessageSizeLimit``

doc/release-notes/11142-more-detailed-file-differences.md

@@ -0,0 +1,3 @@
+The file page version table now shows more detail, e.g. when there are metadata changes or whether a file has been replaced. 
+A bug that causes adding free-form provenance to a file to fail has been fixed.
+See also #11142 and #11145.

doc/release-notes/11159-preview-url-update.md

@@ -0,0 +1,7 @@
+### Preview URL popup updated
+
+The Preview URL popup and related documentation has been updated to give the dataset more information about anonymous access including the names of the dataset fields that will be withheld from the url user and suggesting how to review the url before releasing it. See also #11159 and #11164.
+
+###Bug Fix
+
+Bug which causes users of the Anonymous Review URL to have some metadata of published datasets withheld has been fixed. See #11202

doc/release-notes/openaire_fix.md

@@ -0,0 +1 @@
+bugfix: openaire implementation can now correctly process one or multiple productionPlaces as geolocation

doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json

@@ -0,0 +1,13 @@
+{
+    "fields": [
+        {
+            "typeName": "targetSampleSize",
+            "value": {
+                "targetSampleActualSize": {
+                    "typeName": "targetSampleSizeFormula",
+                    "value": "n = N*X / (X + N – 1)"
+                }
+            }
+        }
+    ]
+}

doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json

@@ -0,0 +1,4 @@
+{
+    "typeName": "journalArticleType",
+    "value": "abstract"
+}

doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld

@@ -1,11 +1,31 @@
 {
 "citation:depositor": "Admin, Dataverse",
 "title": "Test Dataset",
+"socialscience:collectionMode": [
+  "demonstration"
+],
 "subject": "Computer and Information Science",
+"geospatial:geographicCoverage": [
+  {
+    "geospatial:otherGeographicCoverage": "Cambridge"
+  },
+  {
+    "geospatial:otherGeographicCoverage": "Massachusetts"
+  }
+],
 "author": {
   "citation:authorName": "Admin, Dataverse",
   "citation:authorAffiliation": "GDCC"
 },
+"kindOfData": "demonstration data",
+"citation:keyword": [
+  {
+    "citation:keywordValue": "first keyword"
+  },
+  {
+    "citation:keywordValue": "second keyword"
+  }
+],
 "dateOfDeposit": "2020-10-08",
 "citation:distributor": {
   "citation:distributorName": "Demo Dataverse Repository",
@@ -35,5 +55,9 @@
   "title": "http://purl.org/dc/terms/title",
   "citation": "https://dataverse.org/schema/citation/",
   "dvcore": "https://dataverse.org/schema/core#",
-  "schema": "http://schema.org/"
-}}
+  "schema": "http://schema.org/",
+  "geospatial": "dataverse.siteUrl/schema/geospatial#",
+  "socialscience": "dataverse.siteUrl/schema/socialscience#",
+  "kindOfData": "http://rdf-vocabulary.ddialliance.org/discovery#kindOfData"
+  }
+}

doc/sphinx-guides/source/_static/api/transform-oai-ore-jsonld.xq

@@ -0,0 +1,16 @@
+declare option output:method "json";
+
+let $parameters:={ 'method': 'json' }
+for $record in /json
+  let $metadata:=$record/ore_003adescribes
+
+
+  let $json:=
+    <json type="object">
+      {$metadata/*}
+      {$record/_0040context}
+    </json>
+
+
+  return if ($metadata) then
+      file:write("converted.json",$json, $parameters)

doc/sphinx-guides/source/api/auth.rst

@@ -87,22 +87,20 @@ To register a new user who has authenticated via an OIDC provider, the following
 
   curl  -H "Authorization: Bearer $TOKEN" -X POST http://localhost:8080/api/users/register --data '{"termsAccepted":true}'
 
-If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp``` is disabled, it is essential to send a JSON that includes the property ``termsAccepted``` set to true, indicating that you accept the Terms of Use of the installation. Otherwise, you will not be able to create an account. However, if the feature flag is enabled, Terms of Service acceptance is handled by the identity provider, and it is no longer necessary to include the ``termsAccepted``` parameter in the JSON.
-
-In this JSON, we can also include the fields ``position`` or ``affiliation``, in the same way as when we register a user through the Dataverse UI. These fields are optional, and if not provided, they will be persisted as empty in Dataverse.
-
-There is another flag called ``api-bearer-auth-provide-missing-claims`` that can be enabled to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims. However, this flag will only be considered if the ``api-bearer-auth`` feature flag is enabled. If the latter is not enabled, the ``api-bearer-auth-provide-missing-claims`` flag will be ignored.
-
-With the ``api-bearer-auth-provide-missing-claims`` feature flag enabled, you can include the following properties in the request JSON:
+By default, the Bearer token is expected to include the following claims that will be used to create the user account:
 
 - ``username``
 - ``firstName``
 - ``lastName``
 - ``emailAddress``
 
-If properties are provided in the JSON, but corresponding claims already exist in the identity provider, an error will be thrown, outlining the conflicting properties.
+The one parameter required by default is ``termsAccepted`` which must be set to true, indicating that the user has seen and accepted the Terms of Use of the installation. 
+
+If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp`` is enabled (along with the ``api-bearer-auth`` feature flag), Dataverse assumes that the Terms of Service acceptance was handled by the identity provider, e.g. in the OIDC ``consent`` dialog, and the ``termsAccepted`` parameter is not needed.
+
+There is another flag called ``api-bearer-auth-provide-missing-claims`` that can be enabled (along with the ``api-bearer-auth`` feature flag) to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims listed above. If properties are provided in the JSON, but corresponding claims already exist in the identity provider, an error will be thrown, outlining the conflicting properties. Note that supplying missing claims is configured via a separate feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own.
 
-This functionality is included under a feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own.
+In all cases, the submitted JSON can optionally include the fields ``position`` or ``affiliation``, which will be added to the user's Dataverse account. These fields are optional, and if not provided, they will be persisted as empty in Dataverse.
 
 Signed URLs
 -----------