diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7e6995d76d9..3dba7d52109 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -14,7 +14,7 @@ Thank you for contributing to the Dataverse Project through the creation of a bu WARNING: If this is a security issue it should be reported privately to security@dataverse.org More information on bug issues and contributions can be found in the "Contributing to Dataverse" page: -https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md#bug-reportsissues +https://guides.dataverse.org/en/latest/contributor/index.html Please fill out as much of the template as you can. Start below this comment section. @@ -44,7 +44,6 @@ Start below this comment section. **Any related open or closed issues to this bug report?** - **Screenshots:** No matter the issue, screenshots are always welcome. @@ -53,3 +52,7 @@ To add a screenshot, please use one of the following formats and/or methods desc * https://help.github.com/en/articles/file-attachments-on-issues-and-pull-requests * + + +**Are you thinking about creating a pull request for this issue?** +Help is always welcome, is this bug something you or your organization plan to fix? diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d6248537418..7365cb4317c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea or new feature for the Dataverse software! -title: 'Feature Request/Idea:' +title: 'Feature Request:' labels: 'Type: Feature' assignees: '' @@ -11,7 +11,7 @@ assignees: '' Thank you for contributing to the Dataverse Project through the creation of a feature request! More information on ideas/feature requests and contributions can be found in the "Contributing to Dataverse" page: -https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md#ideasfeature-requests +https://guides.dataverse.org/en/latest/contributor/index.html Please fill out as much of the template as you can. Start below this comment section. @@ -34,3 +34,6 @@ Start below this comment section. **Any open or closed issues related to this feature request?** + +**Are you thinking about creating a pull request for this feature?** +Help is always welcome, is this feature something you or your organization plan to implement? diff --git a/.github/ISSUE_TEMPLATE/idea_proposal.md b/.github/ISSUE_TEMPLATE/idea_proposal.md new file mode 100644 index 00000000000..8cb6c7bfafe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/idea_proposal.md @@ -0,0 +1,40 @@ +--- +name: Idea proposal +about: Propose a new idea for discussion to improve the Dataverse software! +title: 'Suggestion:' +labels: 'Type: Suggestion' +assignees: '' + +--- + + + +**Overview of the Suggestion** + + +**What kind of user is the suggestion intended for?** +(Example users roles: API User, Curator, Depositor, Guest, Superuser, Sysadmin) + + +**What inspired this idea?** + + +**What existing behavior do you want changed?** + + +**Any brand new behavior do you want to add to Dataverse?** + + +**Any open or closed issues related to this suggestion?** + + +**Are you thinking about creating a pull request for this issue?** +Help is always welcome, is this idea something you or your organization plan to implement? diff --git a/doc/release-notes/10015-RO-Crate-metadata-file.md b/doc/release-notes/10015-RO-Crate-metadata-file.md deleted file mode 100644 index 4b018a634f7..00000000000 --- a/doc/release-notes/10015-RO-Crate-metadata-file.md +++ /dev/null @@ -1,10 +0,0 @@ -Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. - -From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: -``` -ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" -ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" -``` - -Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. -For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md deleted file mode 100644 index 7ff17f08f4c..00000000000 --- a/doc/release-notes/10022_upload_redirect_without_tagging.md +++ /dev/null @@ -1,5 +0,0 @@ -If your S3 store does not support tagging and gives an error if you configure direct uploads, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For more details see https://dataverse-guide--10029.org.readthedocs.build/en/10029/developers/big-data-support.html#s3-tags #10022 and #10029. - -## New config options - -- dataverse.files..disable-tagging diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md deleted file mode 100644 index 769100c3804..00000000000 --- a/doc/release-notes/10116-incomplete-metadata-label-setting.md +++ /dev/null @@ -1 +0,0 @@ -Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now be shown for draft versions of such datasets and published datasets that the user can edit. This label can also be made invisible for published datasets (regardless of edit rights) with the new option ``dataverse.ui.show-validity-label-when-published`` set to `false`. diff --git a/doc/release-notes/10137-2-add-disable-reason-flag.md b/doc/release-notes/10137-2-add-disable-reason-flag.md deleted file mode 100644 index ee5257466ee..00000000000 --- a/doc/release-notes/10137-2-add-disable-reason-flag.md +++ /dev/null @@ -1,6 +0,0 @@ -## Release Highlights - -### Feature flag to remove the required "reason" field in the "Return To Author" dialog - -A reason field, that is required to not be empty, was added in v6.2. Installations that handle author communications through email or another system may prefer to not be required to use this new field. v6.2 includes a new -disable-return-to-author-reason feature flag that can be enabled to drop the reason field from the dialog and make sending a reason optional in the api/datasets/{id}/returnToAuthor call. diff --git a/doc/release-notes/10236-openapi-definition-endpoint.md b/doc/release-notes/10236-openapi-definition-endpoint.md deleted file mode 100644 index 60492c29d78..00000000000 --- a/doc/release-notes/10236-openapi-definition-endpoint.md +++ /dev/null @@ -1,8 +0,0 @@ -In Dataverse 6.0 Payara was updated, which caused the url `/openapi` to stop working: - -- https://github.com/IQSS/dataverse/issues/9981 -- https://github.com/payara/Payara/issues/6369 - -When it worked in Dataverse 5.x, the `/openapi` output was generated automatically by Payara, but in this release we have switched to OpenAPI output produced by the [SmallRye OpenAPI plugin](https://github.com/smallrye/smallrye-open-api/tree/main/tools/maven-plugin). This gives us finer control over the output. - -For more information, see the section on [OpenAPI](https://dataverse-guide--10328.org.readthedocs.build/en/10328/api/getting-started.html#openapi) in the API Guide. diff --git a/doc/release-notes/10242-add-feature-dv-api b/doc/release-notes/10242-add-feature-dv-api deleted file mode 100644 index 5c786554ff9..00000000000 --- a/doc/release-notes/10242-add-feature-dv-api +++ /dev/null @@ -1 +0,0 @@ -New api endpoints have been added to allow you to add or remove featured collections from a dataverse collection. diff --git a/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md b/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md deleted file mode 100644 index eb3a79dbf25..00000000000 --- a/doc/release-notes/10288-add-term_uri-metadata-in-keyword-block.md +++ /dev/null @@ -1,53 +0,0 @@ -### New keywordTermURI Metadata in keyword Metadata Block - -Adding a new metadata `keywordTermURI` to the `keyword` metadata block to facilitate the integration of controlled vocabulary services, in particular by adding the possibility of saving the "term" and its associated URI. For more information, see #10288 and PR #10371. - -## Upgrade Instructions - -1\. Update the Citation metadata block - -- `wget https://github.com/IQSS/dataverse/releases/download/v6.3/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` - -2\. Update your Solr `schema.xml` to include the new field. - - For details, please see https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#updating-the-solr-schema - - -3\. Reindex Solr. - - Once the schema.xml is updated, Solr must be restarted and a reindex initiated. - For details, see https://guides.dataverse.org/en/latest/admin/solr-search-index.html but here is the reindex command: - - `curl http://localhost:8080/api/admin/index` - - -4\. Run ReExportAll to update dataset metadata exports. Follow the instructions in the [Metadata Export of Admin Guide](https://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). - - -## Notes for Dataverse Installation Administrators - -### Data migration to the new `keywordTermURI` field - -You can migrate your `keywordValue` data containing URIs to the new `keywordTermURI` field. -In case of data migration, view the affected data with the following database query: - -``` -SELECT value FROM datasetfieldvalue dfv -INNER JOIN datasetfield df ON df.id = dfv.datasetfield_id -WHERE df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') -AND value ILIKE 'http%'; -``` - -If you wish to migrate your data, a database update is then necessary: - -``` -UPDATE datasetfield df -SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') -FROM datasetfieldvalue dfv -WHERE dfv.datasetfield_id = df.id -AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') -AND dfv.value ILIKE 'http%'; -``` - -A ['Reindex in Place'](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#reindex-in-place) will be required and ReExportAll will need to be run to update the metadata exports of the dataset. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). \ No newline at end of file diff --git a/doc/release-notes/10316_cvoc_http_headers.md b/doc/release-notes/10316_cvoc_http_headers.md deleted file mode 100644 index 4b557383a2e..00000000000 --- a/doc/release-notes/10316_cvoc_http_headers.md +++ /dev/null @@ -1,5 +0,0 @@ -You are now able to add HTTP request headers required by the External Vocabulary Services you are implementing. - -A combined documentation can be found on pull request [#10404](https://github.com/IQSS/dataverse/pull/10404). - -For more information, see issue [#10316](https://github.com/IQSS/dataverse/issues/10316) and pull request [gddc/dataverse-external-vocab-support#19](https://github.com/gdcc/dataverse-external-vocab-support/pull/19). diff --git a/doc/release-notes/10330-api-change-latest-version-status.md b/doc/release-notes/10330-api-change-latest-version-status.md deleted file mode 100644 index 6e6a018fe12..00000000000 --- a/doc/release-notes/10330-api-change-latest-version-status.md +++ /dev/null @@ -1 +0,0 @@ -The API endpoint for getting the Dataset version has been extended to include latestVersionPublishingStatus. \ No newline at end of file diff --git a/doc/release-notes/10339-workflow.md b/doc/release-notes/10339-workflow.md deleted file mode 100644 index 90d08dabb1f..00000000000 --- a/doc/release-notes/10339-workflow.md +++ /dev/null @@ -1,3 +0,0 @@ -The computational workflow metadata block has been updated to present a clickable link for the External Code Repository URL field. - -Release notes should include the usual instructions, for those who have installed this optional block, to update the computational_workflow block. (PR#10441) \ No newline at end of file diff --git a/doc/release-notes/10389-metadatablocks-api-extension.md b/doc/release-notes/10389-metadatablocks-api-extension.md deleted file mode 100644 index 9b14100d33c..00000000000 --- a/doc/release-notes/10389-metadatablocks-api-extension.md +++ /dev/null @@ -1,6 +0,0 @@ -New optional query parameters added to ``api/metadatablocks`` and ``api/dataverses/{id}/metadatablocks`` endpoints: - -- ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. -- ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. - -Added new ``displayOnCreate`` field to the MetadataBlock and DatasetFieldType payloads. diff --git a/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md b/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md deleted file mode 100644 index e8840e9d4f7..00000000000 --- a/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md +++ /dev/null @@ -1,4 +0,0 @@ -For scenarios involving API calls related to large datasets (Numerous files, for example: ~10k) it has been optimized: - -- The search API endpoint. -- The permission checking logic present in PermissionServiceBean. diff --git a/doc/release-notes/10425-add-MIT-License.md b/doc/release-notes/10425-add-MIT-License.md deleted file mode 100644 index 95d6fb38ded..00000000000 --- a/doc/release-notes/10425-add-MIT-License.md +++ /dev/null @@ -1,3 +0,0 @@ -A new file has been added to import the MIT License to Dataverse: licenseMIT.json. - -Documentation has been added to explain the procedure for adding new licenses to the guides. diff --git a/doc/release-notes/10464-add-name-harvesting-client-facet.md b/doc/release-notes/10464-add-name-harvesting-client-facet.md deleted file mode 100644 index 1fc0bb47caf..00000000000 --- a/doc/release-notes/10464-add-name-harvesting-client-facet.md +++ /dev/null @@ -1,3 +0,0 @@ -The Metadata Source facet has been updated to show the name of the harvesting client rather than grouping all such datasets under 'harvested' - -TODO: for the v6.13 release note: Please add a full re-index using http://localhost:8080/api/admin/index to the upgrade instructions. diff --git a/doc/release-notes/10466-math-challenge-403-error-page.md b/doc/release-notes/10466-math-challenge-403-error-page.md deleted file mode 100644 index 160c760dc9d..00000000000 --- a/doc/release-notes/10466-math-challenge-403-error-page.md +++ /dev/null @@ -1 +0,0 @@ -On forbidden access error page, also know as 403 error page, the math challenge is now correctly display to submit the contact form. diff --git a/doc/release-notes/10468-doc-datalad-integration.md b/doc/release-notes/10468-doc-datalad-integration.md deleted file mode 100644 index cd4d2d53a5f..00000000000 --- a/doc/release-notes/10468-doc-datalad-integration.md +++ /dev/null @@ -1 +0,0 @@ -DataLad has been integrated with Dataverse. For more information, see https://dataverse-guide--10470.org.readthedocs.build/en/10470/admin/integrations.html#datalad diff --git a/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md b/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md deleted file mode 100644 index 77cc7f59773..00000000000 --- a/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md +++ /dev/null @@ -1,3 +0,0 @@ -Changed ``api/dataverses/{id}/metadatablocks`` so that setting the query parameter ``onlyDisplayedOnCreate=true`` also returns metadata blocks with dataset field type input levels configured as required on the General Information page of the collection, in addition to the metadata blocks and their fields with the property ``displayOnCreate=true`` (which was the original behavior). - -A new endpoint ``api/dataverses/{id}/inputLevels`` has been created for updating the dataset field type input levels of a collection via API. diff --git a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md deleted file mode 100644 index 5293c7267d0..00000000000 --- a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md +++ /dev/null @@ -1,22 +0,0 @@ -The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}. -```javascript -{ - "status": "OK", - "data": { - "id": 32, - "alias": "dv6f645bb5", - "name": "dv6f645bb5", - "dataverseContacts": [ - { - "displayOrder": 0, - "contactEmail": "54180268@mailinator.com" - } - ], - "permissionRoot": true, - "dataverseType": "UNCATEGORIZED", - "ownerId": 1, - "creationDate": "2024-04-12T18:05:59Z", - "isReleased": true - } -} -``` \ No newline at end of file diff --git a/doc/release-notes/10494-payara-upgrade.md b/doc/release-notes/10494-payara-upgrade.md deleted file mode 100644 index 23ee0e698f7..00000000000 --- a/doc/release-notes/10494-payara-upgrade.md +++ /dev/null @@ -1,119 +0,0 @@ -# Upgrade Payara to v6.2024.6 - -With this version of Dataverse, we encourage you to upgrade to version 6.2024.6. -This will address security issues accumulated since the release of 6.2023.8, which was required since Dataverse release 6.0. - -## Instructions for Upgrading - -If you are using GDCC containers, this upgrade is included when pulling new release images. -No manual intervention is necessary. - -We recommend you ensure you followed all update instructions from the past releases regarding Payara. -(Latest Payara update was for [v6.0](https://github.com/IQSS/dataverse/releases/tag/v6.0)) - -Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc. - -The steps below are a simple matter of reusing your existing domain directory with the new distribution. -But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: -[Payara Release Notes](https://docs.payara.fish/community/docs/Release%20Notes/Release%20Notes%206.2024.6.html) -We assume you are already on a Dataverse 6.x installation, using a Payara 6.x release. - -```shell -export PAYARA=/usr/local/payara6 -``` - -(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) - -1\. Undeploy the previous version - -```shell - $PAYARA/bin/asadmin list-applications - $PAYARA/bin/asadmin undeploy dataverse<-version> -``` - -2\. Stop Payara - -```shell - service payara stop - rm -rf $PAYARA/glassfish/domains/domain1/generated - rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache - rm -rf $PAYARA/glassfish/domains/domain1/lib/databases -``` - -3\. Move the current Payara directory out of the way - -```shell - mv $PAYARA $PAYARA.MOVED -``` - -4\. Download the new Payara version (6.2024.6), and unzip it in its place - -5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved domain1 - -6\. Make sure that you have the following `--add-opens` options in your domain.xml. If not present, add them: - -```diff ---- payara-6.2023.8/glassfish/domains/domain1/config/domain.xml -+++ payara-6.2024.6/glassfish/domains/domain1/config/domain.xml -@@ -212,12 +212,16 @@ - --add-opens=java.naming/javax.naming.spi=ALL-UNNAMED - --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED - --add-opens=java.logging/java.util.logging=ALL-UNNAMED -+ --add-opens=java.management/javax.management=ALL-UNNAMED -+ --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED - [17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED - [17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED - [17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED - [17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED - [17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED - [17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED -+ [17|]--add-opens=java.base/java.io=ALL-UNNAMED -+ [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED - -Xmx512m - -XX:NewRatio=2 - -XX:+UnlockDiagnosticVMOptions -@@ -447,12 +451,16 @@ - --add-opens=java.naming/javax.naming.spi=ALL-UNNAMED - --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED - --add-opens=java.logging/java.util.logging=ALL-UNNAMED -+ --add-opens=java.management/javax.management=ALL-UNNAMED -+ --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED - [17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED - [17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED - [17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED - [17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED - [17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED - [17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED -+ [17|]--add-opens=java.base/java.io=ALL-UNNAMED -+ [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED - -Xmx512m - -XX:NewRatio=2 - -XX:+UnlockDiagnosticVMOptions -``` -(You can also save this as a patch file and try to apply it.) - -TODO: For the combined 6.3 release note, I would consider replacing the patch format above with just the 4 specific options, for clarity etc. (L.A.) As in: -``` - --add-opens=java.management/javax.management=ALL-UNNAMED - --add-opens=java.management/javax.management.openmbean=ALL-UNNAMED - [17|]--add-opens=java.base/java.io=ALL-UNNAMED - [21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -``` - -7\. Start Payara - -```shell - service payara start -``` - -8\. Deploy this version. - -```shell - $PAYARA/bin/asadmin deploy dataverse-6.3.war -``` - -9\. Restart payara - -```shell - service payara stop - service payara start diff --git a/doc/release-notes/10503-cvoc-hidden-html-fields.md b/doc/release-notes/10503-cvoc-hidden-html-fields.md deleted file mode 100644 index e3ea0463fb8..00000000000 --- a/doc/release-notes/10503-cvoc-hidden-html-fields.md +++ /dev/null @@ -1,11 +0,0 @@ -## Release Highlights - -### Updates on Support for External Vocabulary Services - -#### Hidden HTML Fields - -External Controlled Vocabulary scripts, configured via [:CVocConf](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf), can now access the values of managed fields as well as the term-uri-field for use in constructing the metadata view for a dataset. - -Those values are hidden and can be found with the html attribute `data-cvoc-metadata-name`. - -For more information, see [#10503](https://github.com/IQSS/dataverse/pull/10503). diff --git a/doc/release-notes/10508-base-image-fixes.md b/doc/release-notes/10508-base-image-fixes.md new file mode 100644 index 00000000000..148066435e8 --- /dev/null +++ b/doc/release-notes/10508-base-image-fixes.md @@ -0,0 +1,12 @@ +# Security and Compatibility Fixes to the Container Base Image + +- Switch "wait-for" to "wait4x", aligned with the Configbaker Image +- Update "jattach" to v2.2 +- Install AMD64 / ARM64 versions of tools as necessary +- Run base image as unprivileged user by default instead of `root` - this was an oversight from OpenShift changes +- Linux User, Payara Admin and Domain Master passwords: + - Print hints about default, public knowledge passwords in place for + - Enable replacing these passwords at container boot time +- Enable building with updates Temurin JRE image based on Ubuntu 24.04 LTS +- Fix entrypoint script troubles with pre- and postboot script files +- Unify location of files at CONFIG_DIR=/opt/payara/config, avoid writing to other places \ No newline at end of file diff --git a/doc/release-notes/10531-contrib.md b/doc/release-notes/10531-contrib.md deleted file mode 100644 index 6cfbe988992..00000000000 --- a/doc/release-notes/10531-contrib.md +++ /dev/null @@ -1 +0,0 @@ -A new [Contributor Guide](https://dataverse-guide--10532.org.readthedocs.build/en/10532/contributor/index.html) has been added by the UX Working Group (#10531 and #10532). diff --git a/doc/release-notes/10547-solr-updates.md b/doc/release-notes/10547-solr-updates.md deleted file mode 100644 index a21809c6369..00000000000 --- a/doc/release-notes/10547-solr-updates.md +++ /dev/null @@ -1 +0,0 @@ -Multiple improvements have ben made to they way Solr indexing and searching is done. Response times should be significantly improved. See the individual PRs in this release for details. \ No newline at end of file diff --git a/doc/release-notes/10554-avoid-solr-join-guest.md b/doc/release-notes/10554-avoid-solr-join-guest.md deleted file mode 100644 index 956c658dbed..00000000000 --- a/doc/release-notes/10554-avoid-solr-join-guest.md +++ /dev/null @@ -1,5 +0,0 @@ -Two experimental features flag called "add-publicobject-solr-field" and "avoid-expensive-solr-join" have been added to change how Solr documents are indexed for public objects and how Solr queries are constructed to accommodate access to restricted content (drafts, etc.). It is hoped that it will help with performance, especially on large instances and under load. - -Before the search feature flag ("avoid-expensive...") can be turned on, the indexing flag must be enabled, and a full reindex performed. Otherwise publicly available objects are NOT going to be shown in search results. - -For details see https://dataverse-guide--10555.org.readthedocs.build/en/10555/installation/config.html#feature-flags and #10555. diff --git a/doc/release-notes/10561-3dviewer.md b/doc/release-notes/10561-3dviewer.md deleted file mode 100644 index 47da10f8837..00000000000 --- a/doc/release-notes/10561-3dviewer.md +++ /dev/null @@ -1 +0,0 @@ -3DViewer by openforestdata.pl has been added to the list of external tools: https://preview.guides.gdcc.io/en/develop/admin/external-tools.html#inventory-of-external-tools diff --git a/doc/release-notes/10565-banner-test-improvements.md b/doc/release-notes/10565-banner-test-improvements.md deleted file mode 100644 index d9030f2a0c3..00000000000 --- a/doc/release-notes/10565-banner-test-improvements.md +++ /dev/null @@ -1 +0,0 @@ -The endpoint `api/admin/bannerMessage` has been extended so the ID is returned when created \ No newline at end of file diff --git a/doc/release-notes/10568-Fix File Reingest.md b/doc/release-notes/10568-Fix File Reingest.md deleted file mode 100644 index 354aa847f01..00000000000 --- a/doc/release-notes/10568-Fix File Reingest.md +++ /dev/null @@ -1 +0,0 @@ -A bug that prevented the Ingest option in the File page Edit File menu from working has been fixed \ No newline at end of file diff --git a/doc/release-notes/10570-extra-facet-settings.md b/doc/release-notes/10570-extra-facet-settings.md deleted file mode 100644 index 9d68defc9a3..00000000000 --- a/doc/release-notes/10570-extra-facet-settings.md +++ /dev/null @@ -1,4 +0,0 @@ -Extra settings have been added giving an instance admin more choices in -selectively limiting the availability of search facets on the Collection and Dataset pages. -See the [Disable Solr Facets](https://guides.dataverse.org/en/6.3/installation/config.html#DisableSolrFacets) sections of the Config Guide for more info. - diff --git a/doc/release-notes/10579-avoid-solr-deletes.md b/doc/release-notes/10579-avoid-solr-deletes.md deleted file mode 100644 index 1062a2fb78f..00000000000 --- a/doc/release-notes/10579-avoid-solr-deletes.md +++ /dev/null @@ -1,9 +0,0 @@ -A features flag called "reduce-solr-deletes" has been added to improve how datafiles are indexed. When the flag is enabled, -Dataverse wil avoid pre-emptively deleting existing solr documents for the files prior to sending updated information. This -should improve performance and will allow additional optimizations going forward. - -The /api/admin/index/status and /api/admin/index/clear-orphans calls -(see https://guides.dataverse.org/en/latest/admin/solr-search-index.html#index-and-database-consistency) -will now find and remove (respectively) additional permissions related solr documents that were not being detected before. -Reducing the overall number of documents will improve solr performance and large sites may wish to periodically call the -clear-orphans API. \ No newline at end of file diff --git a/doc/release-notes/10606-dataverse-in-windows-wsl.md b/doc/release-notes/10606-dataverse-in-windows-wsl.md new file mode 100644 index 00000000000..9501d6e3090 --- /dev/null +++ b/doc/release-notes/10606-dataverse-in-windows-wsl.md @@ -0,0 +1 @@ +New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See https://dataverse-guide--10608.org.readthedocs.build/en/10608/developers/windows.html #10606 and #10608. diff --git a/doc/release-notes/10611-harvested-origin-facet.md b/doc/release-notes/10611-harvested-origin-facet.md deleted file mode 100644 index 89ab6eb7639..00000000000 --- a/doc/release-notes/10611-harvested-origin-facet.md +++ /dev/null @@ -1,10 +0,0 @@ -NOTE that this release note supercedes the 10464-add-name-harvesting-client-facet.md note from the PR 10464. - -An option has been added to index the name of the Harvesting Client as the "Metadata Source" of harvested datasets and files; if enabled, the Metadata Source facet will be showing separate entries for the content harvested from different sources, instead of the current, default behavior where there is one "Harvested" facet for all such content. - - -TODO: for the v6.3 release note: -If you choose to enable the extended "Metadata Souce" facet for harvested content, set the optional feature flage (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. - -[Please note that the upgrade instruction in 6.3 will contain a suggestion to run full reindex, as part of the Solr upgrade, so the sentence above will need to be added to that section] - diff --git a/doc/release-notes/5621_dataset image in header.md b/doc/release-notes/5621_dataset image in header.md deleted file mode 100644 index 34b445fd9e1..00000000000 --- a/doc/release-notes/5621_dataset image in header.md +++ /dev/null @@ -1 +0,0 @@ -Dataverse will use the Dataset thumbnail, if one is defined, rather than the generic Dataverse logo in the Open Graph metadata header. This means the image will be seen when, for example, the dataset is referenced in Facebook. diff --git a/doc/release-notes/6.3-release-notes.md b/doc/release-notes/6.3-release-notes.md new file mode 100644 index 00000000000..6cad513690d --- /dev/null +++ b/doc/release-notes/6.3-release-notes.md @@ -0,0 +1,470 @@ +# Dataverse 6.3 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.3 rather than the list of releases, which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +# Table of Contents +- [Release Highlights](#release-highlights) +- [Features](#features) +- [Bug Fixes](#bug-fixes) +- [API](#api) +- [Settings](#settings) +- [Complete List of Changes](#complete-list-of-changes) +- [Getting Help](#getting-help) +- [Upgrade instructions](#upgrade-instructions) + +## Release Highlights + +### Solr Search and Indexing Improvements + +Multiple improvements have been made to the way Solr indexing and searching is done. Response times should be significantly improved. + +- Two experimental features flag called "add-publicobject-solr-field" and "avoid-expensive-solr-join" have been added to change how Solr documents are indexed for public objects and how Solr queries are constructed to accommodate access to restricted content (drafts, etc.). It is hoped that it will help with performance, especially on large instances and under load. + +- Before the search feature flag ("avoid-expensive...") can be turned on, the indexing flag must be enabled, and a full reindex performed. Otherwise publicly available objects are NOT going to be shown in search results. + +- A feature flag called "reduce-solr-deletes" has been added to improve how datafiles are indexed. When the flag is enabled, Dataverse will avoid pre-emptively deleting existing Solr documents for the files prior to sending updated information. This +should improve performance and will allow additional optimizations going forward. + +- The /api/admin/index/status and /api/admin/index/clear-orphans calls +(see https://guides.dataverse.org/en/latest/admin/solr-search-index.html#index-and-database-consistency) +will now find and remove (respectively) additional permissions related Solr documents that were not being detected before. +Reducing the overall number of documents will improve Solr performance and large sites may wish to periodically call the "clear-orphans" API. + +- Dataverse now relies on the autoCommit and autoSoftCommit settings in the Solr configuration instead of explicitly committing documents to the Solr index. This improves indexing speed. + +See also #10554, #10654, and #10579. + +### File Retention Period + +Dataverse now supports file-level retention periods. The ability to set retention periods, with a minimum duration (in months), can be configured by a Dataverse installation administrator. For more information, see the [Retention Periods section](https://guides.dataverse.org/en/6.3/user/dataset-management.html#retention-periods) of the User Guide. + +- Users can configure a specific retention period, defined by an end date and a short reason, on a set of selected files or an individual file, by selecting the "Retention Period" menu item and entering information in a popup dialog. Retention periods can only be set, changed, or removed before a file has been published. After publication, only Dataverse installation administrators can make changes, using an API. + +- After the retention period expires, files can not be previewed or downloaded (as if restricted, with no option to allow access requests). The file (landing) page and all the metadata remains available. + +## Features + +### Large Datasets Improvements + +For scenarios involving API calls related to large datasets (numerous files, for example: ~10k) the following have been been optimized: + +- The Search API endpoint. +- The permission checking logic present in PermissionServiceBean. + +See also [#10415](https://github.com/IQSS/dataverse/pull/10415). + +### Improved Controlled Vocabulary for Citation Block + +The Controlled Vocabuary Values list for the "Language" metadata field in the citation block has been improved, with some missing two- and three-letter ISO 639 codes added, as well as more alternative names for some of the languages, making all these extra language identifiers importable. See also [#8243](https://github.com/IQSS/dataverse/pull/8243). + +### Updates on Support for External Vocabulary Services + +Multiple extensions of the external vocabulary mechanism have been added. These extensions allow interaction with services based on the Ontoportal software and are expected to be generally useful for other service types. + +These changes include: + +- *Improved Indexing with Compound Fields:* When using an external vocabulary service with compound fields, you can now specify which field(s) will include additional indexed information, such as translations of an entry into other languages. This is done by adding the `indexIn` in `retrieval-filtering`. See also [#10505](https://github.com/IQSS/dataverse/pull/10505) and [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). + +- *Broader Support for Indexing Service Responses:* Indexing of the results from `retrieval-filtering` responses can now handle additional formats including JSON arrays of strings and values from arbitrary keys within a JSON Object. See [#10505](https://github.com/IQSS/dataverse/pull/10505). + +- *HTTP Headers:* You are now able to add HTTP request headers required by the service you are implementing. See [#10331](https://github.com/IQSS/dataverse/pull/10331). + +- *Flexible params in retrievalUri:* You can now use `managed-fields` field names as well as the `term-uri-field` field name as parameters in the `retrieval-uri` when configuring an external vocabulary service. `{0}` as an alternative to using the `term-uri-field` name is still supported for backward compatibility. Also you can specify if the value must be url encoded with `encodeUrl:`. See [#10404](https://github.com/IQSS/dataverse/pull/10404). + + For example : `"retrieval-uri": "https://data.agroportal.lirmm.fr/ontologies/{keywordVocabulary}/classes/{encodeUrl:keywordermURL}"` + +- *Hidden HTML Fields* External controlled vocabulary scripts, configured via [:CVocConf](https://guides.dataverse.org/en/6.3/installation/config.html#cvocconf), can now access the values of managed fields as well as the term-uri-field for use in constructing the metadata view for a dataset. These values are now added as hidden elements in the HTML and can be found with the HTML attribute `data-cvoc-metadata-name`. See also [#10503](https://github.com/IQSS/dataverse/pull/10503). + +### A Contributor Guide is now available + +A new [Contributor Guide](https://guides.dataverse.org/en/6.3/contributor/index.html) has been added by the UX Working Group (#10531 and #10532). + +### URL Validation Is More Permissive + +URL validation now allows two slashes in the path component of the URL. +Among other things, this allows metadata fields of `url` type to be filled with more complex url such as https://archive.softwareheritage.org/browse/directory/561bfe6698ca9e58b552b4eb4e56132cac41c6f9/?origin_url=https://github.com/gem-pasteur/macsyfinder&revision=868637fce184865d8e0436338af66a2648e8f6e1&snapshot=1bde3cb370766b10132c4e004c7cb377979928d1 + +See also #9750 and [#9739](https://github.com/IQSS/dataverse/pull/9739) + +### Improved Detection of RO-Crate Files + +Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. + +From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: +``` +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +``` + +Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. +For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate + +See also [#10015](https://github.com/IQSS/dataverse/pull/10015). + +### New S3 Tagging Configuration Option + +If your S3 store does not support tagging and gives an error if you configure direct upload, you can disable the tagging by using the `dataverse.files..disable-tagging` JVM option. For more details, see the section on [S3 tags](https://guides.dataverse.org/en/6.3/developers/big-data-support.html#s3-tags) in the guides, #10022 and #10029. + +### Feature Flag To Remove the Required "Reason" Field in the "Return to Author" Dialog + +A reason field, that is required to not be empty, was added to the "Return to Author" dialog in v6.2. Installations that handle author communications through email or another system may prefer to not be required to use this new field. v6.3 includes a new +disable-return-to-author-reason feature flag that can be enabled to drop the reason field from the dialog and make sending a reason optional in the api/datasets/{id}/returnToAuthor call. See also #10655. + +### Improved Use of Dataverse Thumbnail + +Dataverse will use the dataset thumbnail, if one is defined, rather than the generic Dataverse logo in the Open Graph metadata header. This means the image will be seen when, for example, the dataset is referenced on Facebook. See also [#5621](https://github.com/IQSS/dataverse/pull/5621). + +### Improved Email Notifications When Guestbook is Used for File Access Requests + +Multiple improvements to guestbook response emails making it easier to organize and process them. The subject line of the notification email now includes the name and user identifier of the requestor. Additionally, the body of the email now includes the user id of the requestor. Finally the guestbook responses have been sorted and spaced to improve readability. See also [#10581](https://github.com/IQSS/dataverse/issues/10581). + +### New keywordTermURI Metadata Field in the Citation Metadata Block + +A new metadata field - `keywordTermURI`, has been added in the citation metadata block (as a fourth child field under the `keyword` parent field). This has been done to improve usability and to facilitate the integration of controlled vocabulary services, adding the possibility of saving the "term" and/or its associated URI. For more information, see #10288 and PR #10371. + +### Updated Computational Workflow Metadata Block + +The computational workflow metadata block has been updated to present a clickable link for the External Code Repository URL field. See also [#10339](https://github.com/IQSS/dataverse/pull/10339). + +### Metadata Source Facet Added + +An option has been added to index the name of the harvesting client as the "Metadata Source" of harvested datasets and files; if enabled, the Metadata Source facet will show separate entries for the content harvested from different sources, instead of the current, default behavior where there is one "Harvested" facet for all such content. + +Tho enable this feature, set the optional feature flage (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. + +See also [#10611](https://github.com/IQSS/dataverse/pull/10611) and #10651. + +### Additional Facet Settings + +Extra settings have been added giving an instance admin more choices in selectively limiting the availability of search facets on the collection and dataset pages. + +See [Disable Solr Facets](https://guides.dataverse.org/en/6.3/installation/config.html#DisableSolrFacets) under the configuration section of the Installation Guide for more info as well as [#10570](https://github.com/IQSS/dataverse/pull/10570). + +### Sitemap Now Supports More Than 50k Items + +Dataverse can now handle more than 50,000 items when generating sitemap files, splitting the content across multiple files to comply with the Sitemap protocol. For details, see the [sitemap section](https://guides.dataverse.org/en/6.3/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines) of the Installation Guide. See also [#8936](https://github.com/IQSS/dataverse/pull/8936) and [#10321](https://github.com/IQSS/dataverse/pull/10321). + +### MIT and Apache 2.0 Licenses Added + +New files have been added to import the MIT and Apache 2.0 Licenses to Dataverse: + +- licenseMIT.json +- licenseApache-2.0.json + +Guidance has been added to the [guides](https://guides.dataverse.org/en/6.2/installation/config.html#adding-custom-licenses) to explain the procedure for adding new licenses to Dataverse. + +See also [#10425](https://github.com/IQSS/dataverse/pull/10425). + +### 3D Viewer by Open Forest Data + +3DViewer by openforestdata.pl has been added to the [list of external tools](https://guides.dataverse.org/en/6.3/admin/external-tools.html#inventory-of-external-tools). See also [#10561](https://github.com/IQSS/dataverse/pull/10561). + +### Datalad Integration With Dataverse + +DataLad has been integrated with Dataverse. For more information, see the [integrations](https://guides.dataverse.org/en/6.3/admin/integrations.html#datalad) section of the guides. See also [#10468](https://github.com/IQSS/dataverse/pull/10468). + +### Rsync Support Has Been Deprecated + +Support for rsync has been deprecated. Information has been removed from the guides for rsync and related software such as Data Capture Module (DCM) and Repository Storage Abstraction Layer (RSAL). You can still find this information in [older versions](https://guides.dataverse.org/en/6.2/developers/big-data-support.html#data-capture-module-dcm) of the guides. See [Settings](#database-settings), below, for deprecated settings. See also [#8985](https://github.com/IQSS/dataverse/pull/8985). + +[↑ Table of Contents](#table-of-contents) + +## Bug Fixes + +### OpenAPI Re-Enabled + +In Dataverse 6.0 when Payara was updated it caused the url `/openapi` to stop working: + +- https://github.com/IQSS/dataverse/issues/9981 +- https://github.com/payara/Payara/issues/6369 + +In addition to fixing the `/openapi` URL, we are also making some changes on how we provide the OpenAPI document: + +When it worked in Dataverse 5.x, the `/openapi` output was generated automatically by Payara, but in this release we have switched to OpenAPI output produced by the [SmallRye OpenAPI plugin](https://github.com/smallrye/smallrye-open-api/tree/main/tools/maven-plugin). This gives us finer control over the output. + +For more information, see the section on [OpenAPI](https://guides.dataverse.org/en/6.3/getting-started.html#openapi) in the API Guide and #10328. + +### Re-Addition of "Cell Counting" to Life Sciences Block + +In the Life Sciences metadata block under the "Measurement Type" field the value `cell counting` was accidentally removed in v5.1. It has been restored. See also #8655 and #9735. + +### Math Challenge Fixed on 403 Error Page + +On the "forbidden" (403) error page, the math challenge now correctly displays so that the contact form can be submitted. See also [#10466](https://github.com/IQSS/dataverse/pull/10466). + +### Ingest Option Bug Fixed + +A bug that prevented the "Ingest" option in the file page "Edit File" menu from working has been fixed. See also [#10568](https://github.com/IQSS/dataverse/pull/10568). + +### Incomplete Metadata Bug Fix + +A bug was fixed where the `incomplete metadata` label was being shown for published dataset with incomplete metadata in certain scenarios. This label will now be shown for draft versions of such datasets and published datasets that the user can edit. This label can also be made invisible for published datasets (regardless of edit rights) with the new option ``dataverse.ui.show-validity-label-when-published`` set to `false`. See also [#10116](https://github.com/IQSS/dataverse/pull/10116). + +### Identical Role Error Message + +An error is now correctly reported when an attempt is made to assign an identical role to the same collection, dataset, or file. See also [#9729](https://github.com/IQSS/dataverse/pull/9729) and #10465. + +[↑ Table of Contents](#table-of-contents) + +## API + +### Superuser Endpoint + +The existing API endpoint for toggling the superuser status of a user has been deprecated in favor of a new API endpoint that allows you to explicitly and idempotently set the status as true or false. For details, see the [API Guide](https://guides.dataverse.org/en/6.3/api/native-api.html#set-superuser-status), [#9887](https://github.com/IQSS/dataverse/pull/9887) and [#10440](https://github.com/IQSS/dataverse/pull/10440). + +### New Featured Collections Endpoints + +New API endpoints have been added to allow you to add or remove featured collections from a collection. + +See also the sections on [listing, setting, and removing](https://guides.dataverse.org/en/6.3/api/native-api.html#list-featured-collections-for-a-dataverse-collection) featured collections in the API Guide, [#10242](https://github.com/IQSS/dataverse/pull/10242) and #10459. + +### Dataset Version Endpoint Extended + +The API endpoint for getting the Dataset version has been extended to include latestVersionPublishingStatus. See also [#10330](https://github.com/IQSS/dataverse/pull/10330). + +### New Optional Query Parameters for Metadatablocks Endpoints + +New optional query parameters have been added to `api/metadatablocks` and `api/dataverses/{id}/metadatablocks` endpoints: + +- `returnDatasetFieldTypes`: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. +- Setting the query parameter `onlyDisplayedOnCreate=true` also returns metadata blocks with dataset field type input levels configured as required on the General Information page of the collection, in addition to the metadata blocks and their fields with the property ``displayOnCreate=true``. + +See also [#10389](https://github.com/IQSS/dataverse/pull/10389) + +### Dataverse Payload Includes Release Status + +The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}. See also [#10491](https://github.com/IQSS/dataverse/pull/10491). + +### New Field Type Input Level Endpoint + +A new endpoint ``api/dataverses/{id}/inputLevels`` has been created for updating the dataset field type input levels of a collection via API. See also [#10477](https://github.com/IQSS/dataverse/pull/10477). + +### Banner Message Endpoint Extended + +The endpoint `api/admin/bannerMessage` has been extended so the ID is returned when created. See also [#10565](https://github.com/IQSS/dataverse/pull/10565). + +[↑ Table of Contents](#table-of-contents) + +## Settings + +### Database Settings: + +***New:*** + +- :DisableSolrFacets + +***Deprecated (used with rsync):*** + +- :DataCaptureModuleUrl +- :DownloadMethods +- :LocalDataAccessPath +- :RepositoryStorageAbstractionLayerUrl + +### New Configuration Options + +- `dataverse.files..disable-tagging` +- `dataverse.feature.add-publicobject-solr-field` +- `dataverse.feature.avoid-expensive-solr-join` +- `dataverse.feature.reduce-solr-deletes` +- `dataverse.feature.disable-return-to-author-reason` +- `dataverse.feature.index-harvested-metadata-source` +- `dataverse.ui.show-validity-label-when-published` + +[↑ Table of Contents](#table-of-contents) + +## Complete List of Changes + +For the complete list of code changes in this release, see the [6.3 Milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.3+is%3Aclosed) in GitHub. + +[↑ Table of Contents](#table-of-contents) + +## Getting Help + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +[↑ Table of Contents](#table-of-contents) + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. + +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.2. + +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +In the following commands, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. + +`export PAYARA=/usr/local/payara6` + +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) + +1\. Undeploy the previous version. + +- `$PAYARA/bin/asadmin undeploy dataverse-6.2` + +2\. Stop Payara and remove the following directories: + +```shell +service payara stop +rm -rf $PAYARA/glassfish/domains/domain1/generated +rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +3\. Upgrade Payara to v6.2024.6 + +With this version of Dataverse, we encourage you to upgrade to version 6.2024.6. +This will address security issues accumulated since the release of 6.2023.8. + +Note that if you are using GDCC containers, this upgrade is included when pulling new release images. +No manual intervention is necessary. + +The steps below are a simple matter of reusing your existing domain directory with the new distribution. +But we recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: +[Payara Release Notes](https://docs.payara.fish/community/docs/Release%20Notes/Release%20Notes%206.2024.6.html). +We also recommend you ensure you followed all update instructions from the past releases regarding Payara. +(The latest Payara update was for [v6.0](https://github.com/IQSS/dataverse/releases/tag/v6.0).) + +Move the current Payara directory out of the way: + +```shell +mv $PAYARA $PAYARA.6.2023.8 +``` + +Download the new Payara version 6.2024.6 (from https://www.payara.fish/downloads/payara-platform-community-edition/), and unzip it in its place: + +```shell +cd /usr/local +unzip payara-6.2024.6.zip +``` + +Replace the brand new `payara/glassfish/domains/domain1` with your old, preserved domain1: + +```shell +mv payara6/glassfish/domains/domain1 payara6/glassfish/domains/domain1_DIST +mv payara6-2023.8/glassfish/domains/domain1 payara6/glassfish/domains/ +``` + +Make sure that you have the following `--add-opens` options in your `payara6/glassfish/domains/domain1/config/domain.xml`. If not present, add them: + +``` +--add-opens=java.management/javax.management=ALL-UNNAMED +--add-opens=java.management/javax.management.openmbean=ALL-UNNAMED +[17|]--add-opens=java.base/java.io=ALL-UNNAMED +[21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED +``` + +(Note that you likely already have the `java.base/java.io` option there, but without the `[17|]` prefix. Make sure to replace it with the version above) + +Start Payara: + +```shell +sudo service payara start +``` + +4\. Deploy this version. + +```shell +$PAYARA/bin/asadmin deploy dataverse-6.3.war +``` + +5\. For installations with internationalization: + +- Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +6\. Restart Payara + +```shell +service payara stop +service payara start +``` + +7\. Update the following metadata blocks to reflect the incremental improvements made to the handling of core metadata fields: + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv + +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/biomedical.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file biomedical.tsv + +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/computational_workflow.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file computational_workflow.tsv + +``` + +8\. Upgrade Solr + +Solr 9.4.1 is now the version recommended in our Installation Guide and used with automated testing. There is a known security issue in the previously recommended version 9.3.0: https://nvd.nist.gov/vuln/detail/CVE-2023-36478. While the risk of an exploit should not be significant unless the Solr instance is accessible from outside networks (which we have always recommended against), we recommend to upgrade. + +Install Solr 9.4.1 following the [instructions](https://guides.dataverse.org/en/6.3/installation/prerequisites.html#solr) from the Installation Guide. + +The instructions in the guide suggest to use the config files from the installer zip bundle. Upgrading an existing instance, it may be easier to download them from the source tree: + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/conf/solr/solrconfig.xml +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/conf/solr/schema.xml +cp solrconfig.xml schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf +``` + +8a\. For installations with custom or experimental metadata blocks: + +- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.3/installation/prerequisites.html#solr-init-script)). + +- Run the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the correct path of your Solr installation): + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/conf/solr/update-fields.sh +chmod +x update-fields.sh +curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` + +- Start Solr instance (usually `service solr start` depending on Solr/OS). + +9\. Enable the Metadata Source facet for harvested content (Optional): + +If you choose to enable this new feature, set the optional feature flag (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing. + +10\. Reindex Solr, if you upgraded Solr (recommended), or chose to enable any options that require a reindex: + +```shell +curl http://localhost:8080/api/admin/index +``` + +Note: if you choose to perform a migration of your `keywordValue` metadata fields (section below), that will require a reindex as well, so do that first. + +## Notes for Dataverse Installation Administrators + +### Data migration to the new `keywordTermURI` field + +You can migrate your `keywordValue` data containing URIs to the new `keywordTermURI` field. +In case of data migration, view the affected data with the following database query: + +```sql +SELECT value FROM datasetfieldvalue dfv +INNER JOIN datasetfield df ON df.id = dfv.datasetfield_id +WHERE df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') +AND value ILIKE 'http%'; +``` + +If you wish to migrate your data, a database update is then necessary: + +```sql +UPDATE datasetfield df +SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') +FROM datasetfieldvalue dfv +WHERE dfv.datasetfield_id = df.id +AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') +AND dfv.value ILIKE 'http%'; +``` + +A [reindex in place](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#reindex-in-place) will be required. ReExportAll will need to be run to update the metadata exports of the dataset. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/latest/admin/metadataexport.html#batch-exports-through-the-api). + +[↑ Table of Contents](#table-of-contents) diff --git a/doc/release-notes/8243-improve-language-controlled-vocab.md b/doc/release-notes/8243-improve-language-controlled-vocab.md deleted file mode 100644 index 15b2b46c02d..00000000000 --- a/doc/release-notes/8243-improve-language-controlled-vocab.md +++ /dev/null @@ -1,11 +0,0 @@ -The Controlled Vocabuary Values list for the metadata field Language in the Citation block has been improved, with some missing two- and three-letter ISO 639 codes added, as well as more alternative names for some of the languages, making all these extra language identifiers importable. - -To be added to the 6.3 release instructions: - -Update the Citation block, to incorporate the improved controlled vocabulary for language [plus whatever other improvements may be made to the block in other PRs]: - -``` -wget https://raw.githubusercontent.com/IQSS/dataverse/v6.3/scripts/api/data/metadatablocks/citation.tsv -curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv -``` - diff --git a/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md b/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md deleted file mode 100644 index 295f206871f..00000000000 --- a/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md +++ /dev/null @@ -1,12 +0,0 @@ -## Release Highlights - -### Life Science Metadata - -Re-adding value `cell counting` to Life Science metadatablock's Measurement Type vocabularies accidentally removed in `v5.1`. - -## Upgrade Instructions - -### Update the Life Science metadata block - -- `wget https://github.com/IQSS/dataverse/releases/download/v6.3/biomedical.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` \ No newline at end of file diff --git a/doc/release-notes/8796-fix-license-display-indexing.md b/doc/release-notes/8796-fix-license-display-indexing.md new file mode 100644 index 00000000000..ebded088875 --- /dev/null +++ b/doc/release-notes/8796-fix-license-display-indexing.md @@ -0,0 +1 @@ +When datasets have neither a license nor custom terms of use the display will indicate this. Also, these datasets will no longer be indexed as having custom terms. diff --git a/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md b/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md deleted file mode 100644 index 7b367e328c1..00000000000 --- a/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md +++ /dev/null @@ -1,11 +0,0 @@ -Dataverse can now handle more than 50,000 items when generating sitemap files, splitting the content across multiple files to comply with the Sitemap protocol. - -For details see https://dataverse-guide--10321.org.readthedocs.build/en/10321/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines #8936 and #10321. - -## Upgrade instructions - -If your installation has more than 50,000 entries, you should re-submit your sitemap URL to Google or other search engines. The file in the URL will change from ``sitemap.xml`` to ``sitemap_index.xml``. - -As explained at https://dataverse-guide--10321.org.readthedocs.build/en/10321/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines this is the command for regenerating your sitemap: - -`curl -X POST http://localhost:8080/api/admin/sitemap` diff --git a/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md b/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md new file mode 100644 index 00000000000..145ae5f6d55 --- /dev/null +++ b/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md @@ -0,0 +1,5 @@ +### Shapefile Handling will now ignore files under a hidden directory within the zip file + +Directories that are hidden will be ignored when determining if a zip file contains Shapefile files. + +For more information, see #8945. \ No newline at end of file diff --git a/doc/release-notes/8985-deprecate-rsync.md b/doc/release-notes/8985-deprecate-rsync.md deleted file mode 100644 index 44563f292fd..00000000000 --- a/doc/release-notes/8985-deprecate-rsync.md +++ /dev/null @@ -1,8 +0,0 @@ -Support for rsync has been deprecated. Information has been removed from the guides for rsync and related software such as Data Capture Module (DCM) and Repository Storage Abstraction Layer (RSAL). You can still find this information in [older versions](https://guides.dataverse.org/en/6.2/developers/big-data-support.html#data-capture-module-dcm) of the guides. - -The following related database settings have been deprecated as well: - -- :DataCaptureModuleUrl -- :DownloadMethods -- :LocalDataAccessPath -- :RepositoryStorageAbstractionLayerUrl diff --git a/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md b/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md new file mode 100644 index 00000000000..042b2ec39fd --- /dev/null +++ b/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md @@ -0,0 +1,6 @@ +In an earlier Dataverse release, Datasets with only 'CC0 Waiver' in termsofuse field were converted to 'Custom License' instead of CC0 1.0 licenses during an automated process. A new process was added to correct this. Only Datasets with no terms other than the one create by the previous process will be modified. +- The existing 'Terms of Use' must be equal to 'This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver' +- The following terms fields must be empty: Confidentiality Declaration, Special Permissions, Restrictions, Citation Requirements, Depositor Requirements, Conditions, and Disclaimer. +- The License ID must not be assigned. + +This process will set the License ID to that of the CC0 1.0 license and remove the contents of termsofuse field. diff --git a/doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md b/doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md deleted file mode 100644 index 5e18007e8ae..00000000000 --- a/doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md +++ /dev/null @@ -1,14 +0,0 @@ -## Release Highlights - -### Updates on Support for External Vocabulary Services - -#### HTTP Headers - -You are now able to add HTTP request headers required by the service you are implementing (#10331) - -#### Flexible params in retrievalUri - -You can now use `managed-fields` field names as well as the `term-uri-field` field name as parameters in the `retrieval-uri` when configuring an external vocabulary service. `{0}` as an alternative to using the `term-uri-field` name is still supported for backward compatibility. -Also you can specify if the value must be url encoded with `encodeUrl:`. (#10404) - -For example : `"retrieval-uri": "https://data.agroportal.lirmm.fr/ontologies/{keywordVocabulary}/classes/{encodeUrl:keywordTermURL}"` \ No newline at end of file diff --git a/doc/release-notes/9276-doc-cvoc-index-in.md b/doc/release-notes/9276-doc-cvoc-index-in.md deleted file mode 100644 index 78289201511..00000000000 --- a/doc/release-notes/9276-doc-cvoc-index-in.md +++ /dev/null @@ -1,18 +0,0 @@ -## Release Highlights - -### Updates on Support for External Vocabulary Services - -Multiple extensions of the External Vocabulary mechanism have been added. These extensions allow interaction with services based on the Ontoportal software and are expected to be generally useful for other service types. - -These changes include: - -#### Improved Indexing with Compound Fields - -When using an external vocabulary service with compound fields, you can now specify which field(s) will include additional indexed information, such as translations of an entry into other languages. This is done by adding the `indexIn` in `retrieval-filtering`. (#10505) -For more information, please check [GDCC/dataverse-external-vocab-support documentation](https://github.com/gdcc/dataverse-external-vocab-support/tree/main/docs). - -#### Broader Support for Indexing Service Responses - -Indexing of the results from `retrieval-filtering` responses can now handle additional formats including Json Arrays of Strings and values from arbitrary keys within a JSON Object. (#10505) - -**** This documentation must be merged with 9276-allow-flexible-params-in-retrievaluri-cvoc.md (#10404) \ No newline at end of file diff --git a/doc/release-notes/9375-retention-period.md b/doc/release-notes/9375-retention-period.md deleted file mode 100644 index a088cabf138..00000000000 --- a/doc/release-notes/9375-retention-period.md +++ /dev/null @@ -1,8 +0,0 @@ -The Dataverse Software now supports file-level retention periods. The ability to set retention periods, with a minimum duration (in months), can be configured by a Dataverse installation administrator. For more information, see the [Retention Periods section](https://guides.dataverse.org/en/6.3/user/dataset-management.html#retention-periods) of the Dataverse Software Guides. - -- Users can configure a specific retention period, defined by an end date and a short reason, on a set of selected files or an individual file, by selecting the 'Retention Period' menu item and entering information in a popup dialog. Retention Periods can only be set, changed, or removed before a file has been published. After publication, only Dataverse installation administrators can make changes, using an API. - -- After the retention period expires, files can not be previewed or downloaded (as if restricted, with no option to allow access requests). The file (landing) page and all the metadata remains available. - - -Release notes should mention that a Solr schema update is needed. diff --git a/doc/release-notes/9729-release-notes.md b/doc/release-notes/9729-release-notes.md deleted file mode 100644 index 9dc27995405..00000000000 --- a/doc/release-notes/9729-release-notes.md +++ /dev/null @@ -1 +0,0 @@ -An error is now correctly reported when an attempt is made to assign an identical role to the same collection, dataset, or file. #9729 #10465 \ No newline at end of file diff --git a/doc/release-notes/9739-url-validator.md b/doc/release-notes/9739-url-validator.md deleted file mode 100644 index ad149c54459..00000000000 --- a/doc/release-notes/9739-url-validator.md +++ /dev/null @@ -1,7 +0,0 @@ -## Release Highlights - -### URL validation is more permissive - -Url validation now allows two slashes in the path component of the URL. (#9750) -Among other things, this allows metadata fields of `url` type to be filled with more complex url such as https://archive.softwareheritage.org/browse/directory/561bfe6698ca9e58b552b4eb4e56132cac41c6f9/?origin_url=https://github.com/gem-pasteur/macsyfinder&revision=868637fce184865d8e0436338af66a2648e8f6e1&snapshot=1bde3cb370766b10132c4e004c7cb377979928d1 - diff --git a/doc/release-notes/9887-new-superuser-status-endpoint.md b/doc/release-notes/9887-new-superuser-status-endpoint.md deleted file mode 100644 index 01b1f539f7a..00000000000 --- a/doc/release-notes/9887-new-superuser-status-endpoint.md +++ /dev/null @@ -1 +0,0 @@ -The existing API endpoint for toggling the superuser status of a user has been deprecated in favor of a new API endpoint that allows you to explicitly and idempotently set the status as true or false. For details, see [the guides](https://dataverse-guide--10440.org.readthedocs.build/en/10440/api/native-api.html), #9887 and #10440. \ No newline at end of file diff --git a/doc/release-notes/api-blocking-filter-json.md b/doc/release-notes/api-blocking-filter-json.md new file mode 100644 index 00000000000..337ff82dd8b --- /dev/null +++ b/doc/release-notes/api-blocking-filter-json.md @@ -0,0 +1,3 @@ +* When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. + In case an API client did any special processing to allow it to parse the body, that is no longer necessary. + The status code of such responses has not changed. diff --git a/doc/release-notes/solr-9.4.1.md b/doc/release-notes/solr-9.4.1.md deleted file mode 100644 index 13624a272ab..00000000000 --- a/doc/release-notes/solr-9.4.1.md +++ /dev/null @@ -1,14 +0,0 @@ -Solr 9.4.1 is now the version recommended in our installation guides and used with automated testing. There is a known security issue in the previously recommended version 9.3.0: https://nvd.nist.gov/vuln/detail/CVE-2023-36478. While the risk of an exploit should not be significant unless the Solr instance is accessible from the outside networks (which we have always recommended against), existing Dataverse installations should consider upgrading. - -For the upgrade instructions section: - -[note that 6.3 will contain other solr-related changes, so the instructions may need to contain information merged from multiple release notes!] - -If you are upgrading Solr: - - Install solr-9.4.1 following the instructions from the Installation guide. - - Run a full reindex to populate the search catalog. - - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.4.1` installation directory. This however has not been thoroughly tested and is not officially supported. - - - - diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 2d0dc714132..75ee5a51f90 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -585,6 +585,8 @@ The fully expanded example above (without environment variables) looks like this Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint. +.. _featured-collections: + List Featured Collections for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 6478f15655e..c719fb05e3c 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -68,9 +68,9 @@ # built documents. # # The short X.Y version. -version = '6.2' +version = '6.3' # The full version, including alpha/beta/rc tags. -release = '6.2' +release = '6.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index c41250d48c5..0005265fb1c 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -46,7 +46,7 @@ The base image provides: - CLI tools necessary to run Dataverse (i. e. ``curl`` or ``jq`` - see also :doc:`../installation/prerequisites` in Installation Guide) - Linux tools for analysis, monitoring and so on - `Jattach `__ (attach to running JVM) -- `wait-for `__ (tool to "wait for" a service to be available) +- `wait4x `__ (tool to "wait for" a service to be available) - `dumb-init `__ (see :ref:`below ` for details) This image is created as a "multi-arch image", see :ref:`below `. @@ -85,7 +85,7 @@ Some additional notes, using Maven parameters to change the build and use ...: (See also `Docker Hub search example `_) - ... a different Java Distribution: add ``-Djava.image="name:tag"`` with precise reference to an image available local or remote. -- ... a different UID/GID for the ``payara`` user/group: add ``-Dbase.image.uid=1234`` (or ``.gid``) +- ... a different UID/GID for the ``payara`` user/group (default ``1000:1000``): add ``-Dbase.image.uid=1234`` (or ``.gid``) Automated Builds & Publishing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -151,12 +151,12 @@ provides. These are mostly based on environment variables (very common with cont - [preboot]_ - Abs. path - Provide path to file with ``asadmin`` commands to run **before** boot of application server. - See also `Pre/postboot script docs`_. + See also `Pre/postboot script docs`_. Must be writeable by Payara Linux user! * - ``POSTBOOT_COMMANDS`` - [postboot]_ - Abs. path - Provide path to file with ``asadmin`` commands to run **after** boot of application server. - See also `Pre/postboot script docs`_. + See also `Pre/postboot script docs`_. Must be writeable by Payara Linux user! * - ``JVM_ARGS`` - (empty) - String @@ -231,6 +231,18 @@ provides. These are mostly based on environment variables (very common with cont - See :ref:`:ApplicationServerSettings` ``http.request-timeout-seconds``. *Note:* can also be set using any other `MicroProfile Config Sources`_ available via ``dataverse.http.timeout``. + * - ``PAYARA_ADMIN_PASSWORD`` + - ``admin`` + - String + - Set to secret string to change `Payara Admin Console`_ Adminstrator User ("admin") password. + * - ``LINUX_PASSWORD`` + - ``payara`` + - String + - Set to secret string to change the Payara Linux User ("payara", default UID=1000) password. + * - ``DOMAIN_PASSWORD`` + - ``changeit`` + - String + - Set to secret string to change the `Domain Master Password`_. .. [preboot] ``${CONFIG_DIR}/pre-boot-commands.asadmin`` @@ -374,3 +386,5 @@ from `run-java-sh recommendations`_. .. _Pre/postboot script docs: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Micro%20Documentation/Payara%20Micro%20Configuration%20and%20Management/Micro%20Management/Asadmin%20Commands/Pre%20and%20Post%20Boot%20Commands.html .. _MicroProfile Config Sources: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html .. _run-java-sh recommendations: https://github.com/fabric8io-images/run-java-sh/blob/master/TUNING.md#recommandations +.. _Domain Master Password: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Security%20Guide/Administering%20System%20Security.html#to-change-the-master-password +.. _Payara Admin Console: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/General%20Administration/Overview.html#administration-console \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index e7a59910e56..e436ba9e9d2 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -8,12 +8,12 @@ Making Releases Introduction ------------ -Note: See :doc:`making-library-releases` for how to publish our libraries to Maven Central. - -See :doc:`version-control` for background on our branching strategy. +This document is about releasing the main Dataverse app (https://github.com/IQSS/dataverse). See :doc:`making-library-releases` for how to release our various libraries. Other projects have their own release documentation. The steps below describe making both regular releases and hotfix releases. +Below you'll see branches like "develop" and "master" mentioned. For more on our branching strategy, see :doc:`version-control`. + .. _write-release-notes: Write Release Notes @@ -24,10 +24,10 @@ Developers express the need for an addition to release notes by creating a "rele The task at or near release time is to collect these snippets into a single file. - Create an issue in GitHub to track the work of creating release notes for the upcoming release. -- Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above. -- Delete the release note snippets as the content is added to the main release notes file. -- Include instructions to describe the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. -- Take the release notes .md through the regular Code Review and QA process. +- Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above. Snippets may not include any issue number or pull request number in the text so be sure copy the number from the filename of the snippet into the final release note. +- Delete (``git rm``) the release note snippets as the content is added to the main release notes file. +- Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. +- Take the release notes .md through the regular Code Review and QA process. That is, make a pull request. Create a GitHub Issue and Branch for the Release ------------------------------------------------ @@ -70,6 +70,13 @@ Once important tests have passed (compile, unit tests, etc.), merge the pull req If this is a hotfix release, skip this whole "merge develop to master" step (the "develop" branch is not involved until later). +Add Milestone to Pull Requests and Issues +----------------------------------------- + +Often someone is making sure that the proper milestone (e.g. 5.10.1) is being applied to pull requests and issues, but sometimes this falls between the cracks. + +Check for merged pull requests that have no milestone by going to https://github.com/IQSS/dataverse/pulls and entering `is:pr is:merged no:milestone `_ as a query. If you find any, add the milestone to the pull request and any issues it closes. This includes the "merge develop into master" pull request above. + (Optional) Test Docker Images ----------------------------- @@ -106,7 +113,7 @@ Create a Draft Release on GitHub Go to https://github.com/IQSS/dataverse/releases/new to start creating a draft release. - Under "Choose a tag" you will be creating a new tag. Have it start with a "v" such as ``v5.10.1``. Click "Create new tag on publish". -- Under "Target" go to "Recent Commits" and select the merge commit from when you merged ``develop`` into ``master`` above. This commit will appear in ``/api/info/version`` from a running installation. +- Under "Target", choose "master". This commit will appear in ``/api/info/version`` from a running installation. - Under "Release title" use the same name as the tag such as ``v5.10.1``. - In the description, copy and paste the content from the release notes .md file created in the "Write Release Notes" steps above. - Click "Save draft" because we do not want to publish the release yet. @@ -153,6 +160,7 @@ ssh into the dataverse-internal server and do the following: - ``mkdir target`` - ``cp /tmp/dataverse-5.10.1.war target`` - ``cd scripts/installer`` +- ``make clean`` - ``make`` A zip file called ``dvinstall.zip`` should be produced. @@ -175,7 +183,7 @@ Upload the following artifacts to the draft release you created: Deploy on Demo -------------- -Now that you have the release ready to go, give it one final test by deploying it on https://demo.dataverse.org . Note that this is also an opportunity to re-test the upgrade checklist as described in the release note. +Now that you have the release ready to go, consider giving it one final test by deploying it on https://demo.dataverse.org. Note that this is also an opportunity to re-test the upgrade checklist as described in the release note. Publish the Release ------------------- @@ -194,7 +202,7 @@ ssh into the guides server and update the symlink to point to the latest release cd /var/www/html/en ln -s 5.10.1 latest - +This step could be done before publishing the release if you'd like to double check that links in the release notes work. Close Milestone on GitHub and Create a New One ---------------------------------------------- diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 53578fe980c..699b64c1e1f 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -12,7 +12,102 @@ Running Dataverse in Docker on Windows See the `post `_ by Akio Sone for additional details, but please observe the following: -- In git, the line-ending setting should be set to always LF (line feed, ``core.autocrlf=input``) - You must have jq installed: https://jqlang.github.io/jq/download/ +- In git, the line-ending setting should be set to always LF (line feed, ``core.autocrlf=input``). Update: This should have been fixed by https://github.com/IQSS/dataverse/pull/10092. -One the above is all set you can move on to :doc:`/container/dev-usage` in the Container Guide. +Once the above is all set you can move on to :doc:`/container/dev-usage` in the Container Guide. + +Generally speaking, if you're having trouble running a Dataverse dev environment in Docker on Windows, you are highly encouraged to post about it in the #containers channel on Zulip (https://chat.dataverse.org) and join a Containerization Working Group meeting (https://ct.gdcc.io). See also :doc:`/container/intro` in the Container Guide. + +Running Dataverse in Windows WSL +-------------------------------- + +It is possible to run Dataverse in Windows 10 and 11 through WSL (Windows Subsystem for Linux). + +Please note: these instructions have not been extensively tested. If you find any problems, please open an issue at https://github.com/IQSS/dataverse/issues. + +Install WSL +~~~~~~~~~~~ +If you have Docker already installed, you should already have WSL installed. Otherwise open PowerShell and run: + +.. code-block:: powershell + + wsl --install + +If you already had WSL installed you can install a specific Linux distribution: + +See the list of possible distributions: + +.. code-block:: powershell + + wsl --list --online + +Choose the distribution you would like. Then run the following command. These instructions were tested with Ubuntu. + +.. code-block:: powershell + + wsl --install -d + +You will be asked to create a Linux user. +After the installation of Linux is complete, check that you have an Internet connection: + +.. code-block:: bash + + ping www.google.com + +If you do not have an Internet connection, try adding it in ``/etc/wsl.conf`` + +.. code-block:: bash + + [network] + generateResolvConf = false + +Also in ``/etc/resolv.conf`` add + +.. code-block:: bash + + nameserver 1.1.1.1 + +Now you can install all the tools one usually uses in Linux. For example, it is good idea to run an update: + +.. code-block:: bash + + sudo apt update + sudo apt full-upgrade -y + +Install Dataverse +~~~~~~~~~~~~~~~~~ + +Now you can install Dataverse in WSL following the instructions for :doc:`classic-dev-env` +At the end, check that you have ``-Ddataverse.pid.default-provider=fake`` in jvm-options. + +Now you can access Dataverse in your Windows browser (Edge, Chrome, etc.): + +- http://localhost:8080 +- username: dataverseAdmin +- password: admin + +IDE for Dataverse in Windows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Files in WSL are accessible from Windows for editing using ``\\wsl.localhost`` or ``\\wsl$`` path. Windows files are accessible under Linux in the ``/mnt/c/`` directory. Therefore one can use one's favorite editor or IDE to edit Dataverse project files. Then one can build using ``mvn`` in WSL and deploy manually in WSL using ``asadmin``. + +It is still though possible to use a full-strength IDE. The following instructions are for IntelliJ users. + +- Install Intelij in Windows. + +You can open the project through ``\\wsl.localhost`` and navigate to the Dataverse project. +You can try to build the project in IntelliJ. You may get a message ``Cannot establish network connection from WSL to Windows host (could be blocked by the firewall).`` In that case you can try +to disable WSL Hyperviser from the firewall. +After that you should be able to build the project in IntelliJ. +It seems that at present it is impossible to deploy the Glassfish application in IntelliJ. You can try to add a Glassfish plugin through Settings->Plugins and in Run->Edit Configurations configure Application Server from WSL ``/usr/localhost/payara6`` with URL http://localhost:8080 and Server Domain as domain1, but it may fail since IntelliJ confuses the Windows and Linux paths. + +To use the full strength of Intelij with build, deployment and debugging, one will need to use Intelij ``Remote development``. Close all the projects in IntelliJ and go to ``Remote development->WSL`` and press ``New Project``. In WSL instance choose your Linux distribution and press ``Next``. In ``Project Directory`` navigate to WSL Dataverse project. Then press ``Download IDE and Connect``. This will install IntelliJ in WSL in ``~/.cache/JetBrains/``. Now in IntelliJ you should see your project opened in a new IntelliJ window. After adding the Glassfish plugin and editing your configuration you should be able to build the project and run the project. + +pgAdmin in Windows for Dataverse +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can access the Dataverse database from Windows. + +Install pgAdmin from https://www.pgadmin.org/download/pgadmin-4-windows/ + +In pgAdmin, register a server using 127.0.0.1 with port 5432, database dvndb and dvnapp as username with secret password. Now you will be able to access and update the Dataverse database. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9e4a5e0ee7b..0038c188ea5 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1292,8 +1292,8 @@ Reported Working S3-Compatible Storage Note that for direct uploads and downloads, Dataverse redirects to the proxy-url but presigns the urls based on the ``dataverse.files..custom-endpoint-url``. Also, note that if you choose to enable ``dataverse.files..download-redirect`` the S3 URLs expire after 60 minutes by default. You can change that minute value to reflect a timeout value that’s more appropriate by using ``dataverse.files..url-expiration-minutes``. `Surf Object Store v2019-10-30 `_ - Set ``dataverse.files..payload-signing=true`` and ``dataverse.files..chunked-encoding=false`` to use Surf Object - Store. + Set ``dataverse.files..payload-signing=true``, ``dataverse.files..chunked-encoding=false`` and ``dataverse.files..path-style-request=true`` to use Surf Object + Store. You will need the Swift client (documented at ) to create the access key and secret key for the S3 interface. Note that the ``dataverse.files..proxy-url`` setting can be used in installations where the object store is proxied, but it should be considered an advanced option that will require significant expertise to properly configure. For direct uploads and downloads, Dataverse redirects to the proxy-url but presigns the urls based on the ``dataverse.files..custom-endpoint-url``. @@ -2442,6 +2442,15 @@ Defaults to ``5432``, the default PostgreSQL port. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PORT``. +dataverse.db.parameters ++++++++++++++++++++++++ + +The PostgreSQL server connection parameters. + +Defaults to *empty string* + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PARAMETERS``. + .. _dataverse.solr.host: dataverse.solr.host diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 850702d823e..952eba72616 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.2 +- 6.3 +- `6.2 `__ - `6.1 `__ - `6.0 `__ - `5.14 `__ diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 93f9fa4f0c1..29078e6896c 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -41,11 +41,18 @@ ENV PAYARA_DIR="${HOME_DIR}/appserver" \ STORAGE_DIR="/dv" \ SECRETS_DIR="/secrets" \ DUMPS_DIR="/dumps" \ - PASSWORD_FILE="${HOME_DIR}/passwordFile" \ - ADMIN_USER="admin" \ - ADMIN_PASSWORD="admin" \ + PAYARA_ADMIN_USER="admin" \ + # This is a public default, easy to change via this env var at runtime + PAYARA_ADMIN_PASSWORD="admin" \ DOMAIN_NAME="domain1" \ - PAYARA_ARGS="" + # This is the public default as per https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Security%20Guide/Administering%20System%20Security.html#to-change-the-master-password + # Can be changed at runtime via this env var + DOMAIN_PASSWORD="changeit" \ + PAYARA_ARGS="" \ + LINUX_USER="payara" \ + LINUX_GROUP="payara" \ + # This is a public default and can be changed at runtime using this env var + LINUX_PASSWORD="payara" ENV PATH="${PATH}:${PAYARA_DIR}/bin:${SCRIPT_DIR}" \ DOMAIN_DIR="${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}" \ DEPLOY_PROPS="" \ @@ -69,6 +76,10 @@ ENV PATH="${PATH}:${PAYARA_DIR}/bin:${SCRIPT_DIR}" \ ### PART 1: SYSTEM ### ARG UID=1000 ARG GID=1000 +# Auto-populated by BuildKit / buildx +#ARG TARGETARCH="amd64" +ARG TARGETARCH + USER root WORKDIR / SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -78,23 +89,25 @@ RUN <> /tmp/password-change-file.txt - echo "AS_ADMIN_PASSWORD=${ADMIN_PASSWORD}" >> ${PASSWORD_FILE} - asadmin --user=${ADMIN_USER} --passwordfile=/tmp/password-change-file.txt change-admin-password --domain_name=${DOMAIN_NAME} + echo "AS_ADMIN_NEWPASSWORD=${PAYARA_ADMIN_PASSWORD}" >> /tmp/password-change-file.txt + asadmin --user=${PAYARA_ADMIN_USER} --passwordfile=/tmp/password-change-file.txt change-admin-password --domain_name=${DOMAIN_NAME} + + # Prepare shorthand + PASSWORD_FILE=$(mktemp) + echo "AS_ADMIN_PASSWORD=${PAYARA_ADMIN_PASSWORD}" >> ${PASSWORD_FILE} + ASADMIN="${PAYARA_DIR}/bin/asadmin --user=${PAYARA_ADMIN_USER} --passwordfile=${PASSWORD_FILE}" + # Start domain for configuration ${ASADMIN} start-domain ${DOMAIN_NAME} # Allow access to admin with password only @@ -213,6 +234,7 @@ RUN < "$PASSWORD_FILE" + echo "AS_ADMIN_NEWPASSWORD=${PAYARA_ADMIN_PASSWORD}" >> "$PASSWORD_FILE" + asadmin --user="${PAYARA_ADMIN_USER}" --passwordfile="$PASSWORD_FILE" change-admin-password --domain_name="${DOMAIN_NAME}" + rm "$PASSWORD_FILE" +else + echo "IMPORTANT: THIS CONTAINER USES THE DEFAULT PASSWORD FOR PAYARA ADMIN \"${PAYARA_ADMIN_USER}\"! ('admin')" + echo " To change the password, set the PAYARA_ADMIN_PASSWORD env var." +fi + +# Change the domain master password if necessary +# > The master password is not tied to a user account, and it is not used for authentication. +# > Instead, Payara Server strictly uses the master password to ONLY encrypt the keystore and truststore used to store keys and certificates for the DAS and instances usage. +# It will be requested when booting the application server! +# https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Security%20Guide/Administering%20System%20Security.html#to-change-the-master-password +if [ "$DOMAIN_PASSWORD" != "changeit" ]; then + PASSWORD_FILE=$(mktemp) + echo "AS_ADMIN_MASTERPASSWORD=changeit" >> "$PASSWORD_FILE" + echo "AS_ADMIN_NEWMASTERPASSWORD=${DOMAIN_PASSWORD}" >> "$PASSWORD_FILE" + asadmin --user="${PAYARA_ADMIN_USER}" --passwordfile="$PASSWORD_FILE" change-master-password --savemasterpassword false "${DOMAIN_NAME}" + rm "$PASSWORD_FILE" +else + echo "IMPORTANT: THIS CONTAINER USES THE DEFAULT DOMAIN \"MASTER\" PASSWORD! ('changeit')" + echo " To change the password, set the DOMAIN_PASSWORD env var." +fi diff --git a/modules/container-base/src/main/docker/scripts/init_1_generate_deploy_commands.sh b/modules/container-base/src/main/docker/scripts/init_1_generate_deploy_commands.sh index 161f10caebf..622ea82d6f6 100644 --- a/modules/container-base/src/main/docker/scripts/init_1_generate_deploy_commands.sh +++ b/modules/container-base/src/main/docker/scripts/init_1_generate_deploy_commands.sh @@ -35,12 +35,11 @@ set -euo pipefail # Check required variables are set if [ -z "$DEPLOY_DIR" ]; then echo "Variable DEPLOY_DIR is not set."; exit 1; fi -if [ -z "$PREBOOT_COMMANDS" ]; then echo "Variable PREBOOT_COMMANDS is not set."; exit 1; fi -if [ -z "$POSTBOOT_COMMANDS" ]; then echo "Variable POSTBOOT_COMMANDS is not set."; exit 1; fi - -# Create pre and post boot command files if they don't exist -touch "$POSTBOOT_COMMANDS" -touch "$PREBOOT_COMMANDS" +if [ -z "$PREBOOT_COMMANDS_FILE" ]; then echo "Variable PREBOOT_COMMANDS_FILE is not set."; exit 1; fi +if [ -z "$POSTBOOT_COMMANDS_FILE" ]; then echo "Variable POSTBOOT_COMMANDS_FILE is not set."; exit 1; fi +# Test if files are writeable for us, exit otherwise +touch "$PREBOOT_COMMANDS_FILE" || exit 1 +touch "$POSTBOOT_COMMANDS_FILE" || exit 1 deploy() { @@ -50,14 +49,14 @@ deploy() { fi DEPLOY_STATEMENT="deploy $DEPLOY_PROPS $1" - if grep -q "$1" "$POSTBOOT_COMMANDS"; then - echo "post boot commands already deploys $1"; + if grep -q "$1" "$POSTBOOT_COMMANDS_FILE"; then + echo "Post boot commands already deploys $1, skip adding"; else if [ -n "$SKIP_DEPLOY" ] && { [ "$SKIP_DEPLOY" = "1" ] || [ "$SKIP_DEPLOY" = "true" ]; }; then echo "Skipping deployment of $1 as requested."; else echo "Adding deployment target $1 to post boot commands"; - echo "$DEPLOY_STATEMENT" >> "$POSTBOOT_COMMANDS"; + echo "$DEPLOY_STATEMENT" >> "$POSTBOOT_COMMANDS_FILE"; fi fi } diff --git a/modules/container-base/src/main/docker/scripts/init_1_generate_devmode_commands.sh b/modules/container-base/src/main/docker/scripts/init_1_generate_devmode_commands.sh index 016151168d5..608113d1cf7 100644 --- a/modules/container-base/src/main/docker/scripts/init_1_generate_devmode_commands.sh +++ b/modules/container-base/src/main/docker/scripts/init_1_generate_devmode_commands.sh @@ -11,39 +11,49 @@ set -euo pipefail # for the parent shell before executing Payara. ###### ###### ###### ###### ###### ###### ###### ###### ###### ###### ###### +if [ -z "$PREBOOT_COMMANDS_FILE" ]; then echo "Variable PREBOOT_COMMANDS_FILE is not set."; exit 1; fi +# Test if preboot file is writeable for us, exit otherwise +touch "$PREBOOT_COMMANDS_FILE" || exit 1 + # 0. Init variables ENABLE_JMX=${ENABLE_JMX:-0} ENABLE_JDWP=${ENABLE_JDWP:-0} ENABLE_RELOAD=${ENABLE_RELOAD:-0} -DV_PREBOOT=${CONFIG_DIR}/dataverse_preboot -echo "# Dataverse preboot configuration for Payara" > "${DV_PREBOOT}" +function inject() { + if [ -z "$1" ]; then echo "No line specified"; exit 1; fi + # If the line is not yet in the file, try to add it + if ! grep -q "$1" "$PREBOOT_COMMANDS_FILE"; then + # Check if the line is still not in the file when splitting at the first = + if ! grep -q "$(echo "$1" | cut -f1 -d"=")" "$PREBOOT_COMMANDS_FILE"; then + echo "$1" >> "$PREBOOT_COMMANDS_FILE" + fi + fi +} # 1. Configure JMX (enabled by default on port 8686, but requires SSL) # See also https://blog.payara.fish/monitoring-payara-server-with-jconsole # To still use it, you can use a sidecar container proxying or using JMX via localhost without SSL. if [ "${ENABLE_JMX}" = "1" ]; then echo "Enabling unsecured JMX on 0.0.0.0:8686, enabling AMX and tuning monitoring levels to HIGH. You'll need a sidecar for this, as access is allowed from same machine only (without SSL)." - { \ - echo "set configs.config.server-config.amx-configuration.enabled=true" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.jvm=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.connector-service=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.connector-connection-pool=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.jdbc-connection-pool=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.web-services-container=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.ejb-container=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.security=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.jms-service=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.jersey=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.transaction-service=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.jpa=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.web-container=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.orb=HIGH" - echo "set configs.config.server-config.monitoring-service.module-monitoring-levels.deployment=HIGH" - echo "set configs.config.server-config.admin-service.jmx-connector.system.security-enabled=false" - } >> "${DV_PREBOOT}" + inject "set configs.config.server-config.amx-configuration.enabled=true" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.jvm=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.connector-service=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.connector-connection-pool=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.jdbc-connection-pool=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.web-services-container=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.ejb-container=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.security=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.jms-service=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.jersey=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.transaction-service=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.jpa=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.web-container=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.orb=HIGH" + inject "set configs.config.server-config.monitoring-service.module-monitoring-levels.deployment=HIGH" + inject "set configs.config.server-config.admin-service.jmx-connector.system.security-enabled=false" fi # 2. Enable JDWP via debugging switch @@ -55,17 +65,12 @@ fi # 3. Enable hot reload if [ "${ENABLE_RELOAD}" = "1" ]; then echo "Enabling hot reload of deployments." - echo "set configs.config.server-config.admin-service.das-config.dynamic-reload-enabled=true" >> "${DV_PREBOOT}" - echo "set configs.config.server-config.admin-service.das-config.autodeploy-enabled=true" >> "${DV_PREBOOT}" - export DATAVERSE_JSF_PROJECT_STAGE=${DATAVERSE_JSF_PROJECT_STAGE:-"Development"} - export DATAVERSE_JSF_REFRESH_PERIOD=${DATAVERSE_JSF_REFRESH_PERIOD:-"0"} + inject "set configs.config.server-config.admin-service.das-config.dynamic-reload-enabled=true" + inject "set configs.config.server-config.admin-service.das-config.autodeploy-enabled=true" fi # 4. Add the commands to the existing preboot file, but insert BEFORE deployment -TMP_PREBOOT=$(mktemp) -cat "${DV_PREBOOT}" "${PREBOOT_COMMANDS}" > "${TMP_PREBOOT}" -mv "${TMP_PREBOOT}" "${PREBOOT_COMMANDS}" -echo "DEBUG: preboot contains the following commands:" +echo "DEBUG: preboot contains now the following commands:" +echo "--------------------------------------------------" +cat "${PREBOOT_COMMANDS_FILE}" echo "--------------------------------------------------" -cat "${PREBOOT_COMMANDS}" -echo "--------------------------------------------------" \ No newline at end of file diff --git a/modules/container-base/src/main/docker/scripts/startInForeground.sh b/modules/container-base/src/main/docker/scripts/startInForeground.sh index 4843f6ae055..fa7d533b0d1 100644 --- a/modules/container-base/src/main/docker/scripts/startInForeground.sh +++ b/modules/container-base/src/main/docker/scripts/startInForeground.sh @@ -32,10 +32,11 @@ ########################################################################################################## # Check required variables are set -if [ -z "$ADMIN_USER" ]; then echo "Variable ADMIN_USER is not set."; exit 1; fi -if [ -z "$PASSWORD_FILE" ]; then echo "Variable PASSWORD_FILE is not set."; exit 1; fi -if [ -z "$PREBOOT_COMMANDS" ]; then echo "Variable PREBOOT_COMMANDS is not set."; exit 1; fi -if [ -z "$POSTBOOT_COMMANDS" ]; then echo "Variable POSTBOOT_COMMANDS is not set."; exit 1; fi +if [ -z "$PAYARA_ADMIN_USER" ]; then echo "Variable ADMIN_USER is not set."; exit 1; fi +if [ -z "$PAYARA_ADMIN_PASSWORD" ]; then echo "Variable ADMIN_PASSWORD is not set."; exit 1; fi +if [ -z "$DOMAIN_PASSWORD" ]; then echo "Variable DOMAIN_PASSWORD is not set."; exit 1; fi +if [ -z "$PREBOOT_COMMANDS_FILE" ]; then echo "Variable PREBOOT_COMMANDS_FILE is not set."; exit 1; fi +if [ -z "$POSTBOOT_COMMANDS_FILE" ]; then echo "Variable POSTBOOT_COMMANDS_FILE is not set."; exit 1; fi if [ -z "$DOMAIN_NAME" ]; then echo "Variable DOMAIN_NAME is not set."; exit 1; fi # Check if dumps are enabled - add arg to JVM_ARGS in this case @@ -43,6 +44,13 @@ if [ -n "${ENABLE_DUMPS}" ] && [ "${ENABLE_DUMPS}" = "1" ]; then JVM_ARGS="${JVM_DUMPS_ARG} ${JVM_ARGS}" fi +# For safety reasons, do no longer expose the passwords - malicious code could extract it! +# (We need to save the master password for booting the server though) +MASTER_PASSWORD="${DOMAIN_PASSWORD}" +export LINUX_PASSWORD="have-some-scrambled-eggs" +export PAYARA_ADMIN_PASSWORD="have-some-scrambled-eggs" +export DOMAIN_PASSWORD="have-some-scrambled-eggs" + # The following command gets the command line to be executed by start-domain # - print the command line to the server with --dry-run, each argument on a separate line # - remove -read-string argument @@ -50,19 +58,25 @@ fi # - remove lines before and after the command line and squash commands on a single line # Create pre and post boot command files if they don't exist -touch "$POSTBOOT_COMMANDS" -touch "$PREBOOT_COMMANDS" +touch "$POSTBOOT_COMMANDS_FILE" || exit 1 +touch "$PREBOOT_COMMANDS_FILE" || exit 1 +# This workaround is necessary due to limitations of asadmin +PASSWORD_FILE=$(mktemp) +echo "AS_ADMIN_MASTERPASSWORD=$MASTER_PASSWORD" > "$PASSWORD_FILE" # shellcheck disable=SC2068 # -- Using $@ is necessary here as asadmin cannot deal with options enclosed in ""! -OUTPUT=$("${PAYARA_DIR}"/bin/asadmin --user="${ADMIN_USER}" --passwordfile="${PASSWORD_FILE}" start-domain --dry-run --prebootcommandfile="${PREBOOT_COMMANDS}" --postbootcommandfile="${POSTBOOT_COMMANDS}" $@ "$DOMAIN_NAME") +OUTPUT=$("${PAYARA_DIR}"/bin/asadmin --user="${PAYARA_ADMIN_USER}" --passwordfile="$PASSWORD_FILE" start-domain --dry-run --prebootcommandfile="${PREBOOT_COMMANDS_FILE}" --postbootcommandfile="${POSTBOOT_COMMANDS_FILE}" $@ "$DOMAIN_NAME") STATUS=$? +rm "$PASSWORD_FILE" if [ "$STATUS" -ne 0 ] then echo ERROR: "$OUTPUT" >&2 exit 1 fi +echo "Booting now..." + COMMAND=$(echo "$OUTPUT"\ | sed -n -e '2,/^$/p'\ | sed "s|glassfish.jar|glassfish.jar $JVM_ARGS |g") @@ -72,18 +86,6 @@ echo "$COMMAND" | tr ' ' '\n' echo # Run the server in foreground - read master password from variable or file or use the default "changeit" password - -set +x -if test "$AS_ADMIN_MASTERPASSWORD"x = x -a -f "$PASSWORD_FILE" - then - # shellcheck disable=SC1090 - source "$PASSWORD_FILE" -fi -if test "$AS_ADMIN_MASTERPASSWORD"x = x - then - AS_ADMIN_MASTERPASSWORD=changeit -fi -echo "AS_ADMIN_MASTERPASSWORD=$AS_ADMIN_MASTERPASSWORD" > /tmp/masterpwdfile # shellcheck disable=SC2086 # -- Unquoted exec var is necessary, as otherwise things get escaped that may not be escaped (parameters for Java) -exec ${COMMAND} < /tmp/masterpwdfile +exec ${COMMAND} < <(echo "AS_ADMIN_MASTERPASSWORD=$MASTER_PASSWORD") diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index dae4a3aa272..351425a17ba 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -21,7 +21,7 @@ ENV SCRIPT_DIR="/scripts" \ ENV PATH="${PATH}:${SCRIPT_DIR}" \ BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap" -ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x ed postgresql-client" +ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x ed postgresql-client aws-cli" RUN true && \ # Install necessary software and tools diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 4a67e301f5b..62efbf62317 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -131,7 +131,7 @@ - 6.2 + 6.3 17 UTF-8 diff --git a/src/main/docker/scripts/init_2_configure.sh b/src/main/docker/scripts/init_2_configure.sh index b31cfac37b7..5c1075f01f3 100755 --- a/src/main/docker/scripts/init_2_configure.sh +++ b/src/main/docker/scripts/init_2_configure.sh @@ -22,17 +22,35 @@ if [ "${dataverse_files_storage__driver__id}" = "local" ]; then export dataverse_files_local_directory="${dataverse_files_local_directory:-${STORAGE_DIR}/store}" fi -# 0. Define postboot commands file to be read by Payara and clear it -DV_POSTBOOT=${PAYARA_DIR}/dataverse_postboot -echo "# Dataverse postboot configuration for Payara" > "${DV_POSTBOOT}" +# If reload is enable via ENABLE_RELOAD=1, set according Jakarta Faces options +ENABLE_RELOAD=${ENABLE_RELOAD:-0} +if [ "${ENABLE_RELOAD}" = "1" ]; then + export DATAVERSE_JSF_PROJECT_STAGE=${DATAVERSE_JSF_PROJECT_STAGE:-"Development"} + export DATAVERSE_JSF_REFRESH_PERIOD=${DATAVERSE_JSF_REFRESH_PERIOD:-"0"} +fi + +# Check prerequisites for commands handling +if [ -z "$POSTBOOT_COMMANDS_FILE" ]; then echo "Variable POSTBOOT_COMMANDS_FILE is not set."; exit 1; fi +# Test if postboot file is writeable for us, exit otherwise +touch "$POSTBOOT_COMMANDS_FILE" || exit 1 +# Copy and split the postboot contents to manipulate them +EXISTING_DEPLOY_COMMANDS=$(mktemp) +NEW_POSTBOOT_COMMANDS=$(mktemp) +grep -e "^deploy " "$POSTBOOT_COMMANDS_FILE" > "$EXISTING_DEPLOY_COMMANDS" || true +grep -v -e "^deploy" "$POSTBOOT_COMMANDS_FILE" > "$NEW_POSTBOOT_COMMANDS" || true -# 2. Domain-spaced resources (JDBC, JMS, ...) -# TODO: This is ugly and dirty. It should be replaced with resources from -# EE 8 code annotations or at least glassfish-resources.xml -# NOTE: postboot commands is not multi-line capable, thus spaghetti needed. +function inject() { + if [ -z "$1" ]; then echo "No line specified"; exit 1; fi + # If the line is not yet in the file, try to add it + if ! grep -q "$1" "$NEW_POSTBOOT_COMMANDS"; then + # Check if the line is still not in the file when splitting at the first = + if ! grep -q "$(echo "$1" | cut -f1 -d"=")" "$NEW_POSTBOOT_COMMANDS"; then + echo "$1" >> "$NEW_POSTBOOT_COMMANDS" + fi + fi +} -# 3. Domain based configuration options -# Set Dataverse environment variables +# Domain based configuration options - set from Dataverse environment variables echo "INFO: Defining system properties for Dataverse configuration options." #env | grep -Ee "^(dataverse|doi)_" | sort -fd env -0 | grep -z -Ee "^(dataverse|doi)_" | while IFS='=' read -r -d '' k v; do @@ -47,14 +65,12 @@ env -0 | grep -z -Ee "^(dataverse|doi)_" | while IFS='=' read -r -d '' k v; do v=$(echo "${v}" | sed -e 's/:/\\\:/g') echo "DEBUG: Handling ${KEY}=${v}." - echo "create-system-properties ${KEY}=${v}" >> "${DV_POSTBOOT}" + inject "create-system-properties ${KEY}=${v}" done # 4. Add the commands to the existing postboot file, but insert BEFORE deployment -TMPFILE=$(mktemp) -cat "${DV_POSTBOOT}" "${POSTBOOT_COMMANDS}" > "${TMPFILE}" && mv "${TMPFILE}" "${POSTBOOT_COMMANDS}" +cat "$NEW_POSTBOOT_COMMANDS" "$EXISTING_DEPLOY_COMMANDS" > "${POSTBOOT_COMMANDS_FILE}" echo "DEBUG: postboot contains the following commands:" echo "--------------------------------------------------" -cat "${POSTBOOT_COMMANDS}" +cat "${POSTBOOT_COMMANDS_FILE}" echo "--------------------------------------------------" - diff --git a/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh b/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh index c234ad33307..06b41d60507 100644 --- a/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh +++ b/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh @@ -1,4 +1,4 @@ #It was reported on 9949 that on the first launch of the containers Dataverse would not be deployed on payara #this was caused by a race condition due postgress not being ready. A solion for docker compose was prepared #but didn't work due a compatibility issue on the Maven pluggin [https://github.com/fabric8io/docker-maven-plugin/issues/888] -wait-for "${DATAVERSE_DB_HOST:-postgres}:${DATAVERSE_DB_PORT:-5432}" -t 120 \ No newline at end of file +wait4x tcp "${DATAVERSE_DB_HOST:-postgres}:${DATAVERSE_DB_PORT:-5432}" -t 120s diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index 31e7758c7d5..4bf6c00f199 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -54,15 +54,18 @@ public int compare(DatasetField o1, DatasetField o2) { o2.getDatasetFieldType().getDisplayOrder() ); }}; - public static DatasetField createNewEmptyDatasetField(DatasetFieldType dsfType, Object dsv) { + public static DatasetField createNewEmptyDatasetField(DatasetFieldType dsfType, DatasetVersion dsv) { DatasetField dsfv = createNewEmptyDatasetField(dsfType); - //TODO - a better way to handle this? - if (dsv.getClass().getName().equals("edu.harvard.iq.dataverse.DatasetVersion")){ - dsfv.setDatasetVersion((DatasetVersion)dsv); - } else { - dsfv.setTemplate((Template)dsv); - } + dsfv.setDatasetVersion(dsv); + + return dsfv; + } + + public static DatasetField createNewEmptyDatasetField(DatasetFieldType dsfType, Template dsv) { + + DatasetField dsfv = createNewEmptyDatasetField(dsfType); + dsfv.setTemplate(dsv); return dsfv; } @@ -545,9 +548,12 @@ public String toString() { return "edu.harvard.iq.dataverse.DatasetField[ id=" + id + " ]"; } - public DatasetField copy(Object version) { + public DatasetField copy(DatasetVersion version) { return copy(version, null); } + public DatasetField copy(Template template) { + return copy(template, null); + } // originally this was an overloaded method, but we renamed it to get around an issue with Bean Validation // (that looked t overloaded methods, when it meant to look at overriden methods @@ -555,15 +561,15 @@ public DatasetField copyChild(DatasetFieldCompoundValue parent) { return copy(null, parent); } - private DatasetField copy(Object version, DatasetFieldCompoundValue parent) { + private DatasetField copy(Object versionOrTemplate, DatasetFieldCompoundValue parent) { DatasetField dsf = new DatasetField(); dsf.setDatasetFieldType(datasetFieldType); - if (version != null) { - if (version.getClass().getName().equals("edu.harvard.iq.dataverse.DatasetVersion")) { - dsf.setDatasetVersion((DatasetVersion) version); + if (versionOrTemplate != null) { + if (versionOrTemplate instanceof DatasetVersion) { + dsf.setDatasetVersion((DatasetVersion) versionOrTemplate); } else { - dsf.setTemplate((Template) version); + dsf.setTemplate((Template) versionOrTemplate); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index ab23fa779d5..f99b3ee1b53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -315,6 +315,23 @@ private void msg(String s){ //logger.fine(s); } + public boolean isVersionDefaultCustomTerms(DatasetVersion datasetVersion) { + + if (datasetVersion.getId() != null) { + try { + TermsOfUseAndAccess toua = (TermsOfUseAndAccess) em.createNamedQuery("TermsOfUseAndAccess.findByDatasetVersionIdAndDefaultTerms") + .setParameter("id", datasetVersion.getId()).setParameter("defaultTerms", TermsOfUseAndAccess.DEFAULT_NOTERMS).getSingleResult(); + if (toua != null && datasetVersion.getTermsOfUseAndAccess().getLicense() == null) { + return true; + } + + } catch (NoResultException e) { + return false; + } + } + return false; + } + /** * Does the version identifier in the URL ask for a "DRAFT"? * diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 78b1827c798..978c716e058 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -412,12 +412,18 @@ public List getDataverseFieldTypeInputLevels() { } public boolean isDatasetFieldTypeRequiredAsInputLevel(Long datasetFieldTypeId) { - for(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel : dataverseFieldTypeInputLevels) { - if (dataverseFieldTypeInputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && dataverseFieldTypeInputLevel.isRequired()) { - return true; - } - } - return false; + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && inputLevel.isRequired()); + } + + public boolean isDatasetFieldTypeIncludedAsInputLevel(Long datasetFieldTypeId) { + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && inputLevel.isInclude()); + } + + public boolean isDatasetFieldTypeInInputLevels(Long datasetFieldTypeId) { + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId)); } public Template getDefaultTemplate() { diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java index c53df93def8..4276eb02882 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponsesPage.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.List; import java.util.logging.Logger; @@ -101,8 +102,9 @@ public String init() { private String getFileName(){ // The fix below replaces any spaces in the name of the dataverse with underscores; // without it, the filename was chopped off (by the browser??), and the user - // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. - return dataverse.getName().replace(' ', '_') + "_" + guestbook.getId() + "_GuestbookReponses.csv"; + // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. + // Also removing some chars that have been reported to cause issues with certain browsers + return FileUtil.sanitizeFileName(dataverse.getName() + "_" + guestbook.getId() + "_GuestbookResponses.csv"); } public void streamResponsesByDataverseAndGuestbook(){ diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java index cc89cfd9d56..d1cc515fd01 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageGuestbooksPage.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseGuestbookRootCommand; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import java.util.LinkedList; @@ -220,7 +221,8 @@ private String getFileName(){ // The fix below replaces any spaces in the name of the dataverse with underscores; // without it, the filename was chopped off (by the browser??), and the user // was getting the file name "Foo", instead of "Foo and Bar in Social Sciences.csv". -- L.A. - return dataverse.getName().replace(' ', '_') + "_GuestbookReponses.csv"; + // Also removing some chars that have been reported to cause issues with certain browsers + return FileUtil.sanitizeFileName(dataverse.getName() + "_GuestbookResponses.csv"); } public void deleteGuestbook() { diff --git a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java index ee865770dbe..9e48c6c0165 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java @@ -17,6 +17,28 @@ import jakarta.persistence.Transient; import edu.harvard.iq.dataverse.license.License; +import jakarta.persistence.NamedQueries; +import jakarta.persistence.NamedQuery; + +@NamedQueries({ + // TermsOfUseAndAccess.findByDatasetVersionIdAndDefaultTerms + // is used to determine if the dataset terms were set by the multi license support update + // as part of the 5.10 release. + + @NamedQuery(name = "TermsOfUseAndAccess.findByDatasetVersionIdAndDefaultTerms", + query = "SELECT o FROM TermsOfUseAndAccess o, DatasetVersion dv WHERE " + + "dv.id =:id " + + "AND dv.termsOfUseAndAccess.id = o.id " + + "AND o.termsOfUse =:defaultTerms " + + "AND o.confidentialityDeclaration IS null " + + "AND o.specialPermissions IS null " + + "AND o.restrictions IS null " + + "AND o.citationRequirements IS null " + + "AND o.depositorRequirements IS null " + + "AND o.conditions IS null " + + "AND o.disclaimer IS null " + ) +}) /** * @@ -26,6 +48,8 @@ @Entity @ValidateTermsOfUseAndAccess public class TermsOfUseAndAccess implements Serializable { + + public static final String DEFAULT_NOTERMS = "This dataset is made available without information on how it can be used. You should communicate with the Contact(s) specified before use."; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java b/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java index 0e5b8226310..b51b1aa2612 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java @@ -49,7 +49,7 @@ public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) thro @Override public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) throws IOException, ServletException { HttpServletResponse httpResponse = (HttpServletResponse) sr1; - httpResponse.getWriter().println("{ status:\"error\", message:\"Endpoint blocked. Please contact the dataverse administrator\"}" ); + httpResponse.getWriter().println("{ \"status\":\"error\", \"message\":\"Endpoint blocked. Please contact the dataverse administrator\"}" ); httpResponse.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); httpResponse.setContentType("application/json"); } @@ -67,7 +67,7 @@ public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) thro fc.doFilter(sr, sr1); } else { HttpServletResponse httpResponse = (HttpServletResponse) sr1; - httpResponse.getWriter().println("{ status:\"error\", message:\"Endpoint available from localhost only. Please contact the dataverse administrator\"}" ); + httpResponse.getWriter().println("{ \"status\":\"error\", \"message\":\"Endpoint available from localhost only. Please contact the dataverse administrator\"}" ); httpResponse.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); httpResponse.setContentType("application/json"); } @@ -102,7 +102,7 @@ public void doBlock(ServletRequest sr, ServletResponse sr1, FilterChain fc) thro if ( block ) { HttpServletResponse httpResponse = (HttpServletResponse) sr1; - httpResponse.getWriter().println("{ status:\"error\", message:\"Endpoint available using API key only. Please contact the dataverse administrator\"}" ); + httpResponse.getWriter().println("{ \"status\":\"error\", \"message\":\"Endpoint available using API key only. Please contact the dataverse administrator\"}" ); httpResponse.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); httpResponse.setContentType("application/json"); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 98bd26b51d6..060b8694e9c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -653,6 +653,15 @@ public static License getLicense(DatasetVersion dsv) { } public static String getLicenseName(DatasetVersion dsv) { + + DatasetVersionServiceBean datasetVersionService = CDI.current().select(DatasetVersionServiceBean.class).get(); + /* + Special case where there are default custom terms indicating that no actual choice has been made... + */ + if (datasetVersionService.isVersionDefaultCustomTerms(dsv)) { + return BundleUtil.getStringFromBundle("license.none.chosen"); + } + License license = DatasetUtil.getLicense(dsv); return getLocalizedLicenseName(license); } @@ -683,7 +692,16 @@ public static String getLicenseIcon(DatasetVersion dsv) { } public static String getLicenseDescription(DatasetVersion dsv) { + + DatasetVersionServiceBean datasetVersionService = CDI.current().select(DatasetVersionServiceBean.class).get(); + /* + Special case where there are default custom terms indicating that no actual choice has been made... + */ + if (datasetVersionService.isVersionDefaultCustomTerms(dsv)) { + return BundleUtil.getStringFromBundle("license.none.chosen.description"); + } License license = DatasetUtil.getLicense(dsv); + return license != null ? getLocalizedLicenseDetails(license,"DESCRIPTION") : BundleUtil.getStringFromBundle("license.custom.description"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 26b42734d19..c91eb0bfa7c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -100,6 +100,8 @@ public class IndexServiceBean { @EJB DatasetServiceBean datasetService; @EJB + DatasetVersionServiceBean datasetVersionService; + @EJB BuiltinUserServiceBean dataverseUserServiceBean; @EJB PermissionServiceBean permissionService; @@ -1840,9 +1842,19 @@ private List getDataversePathsFromSegments(List dataversePathSeg private void addLicenseToSolrDoc(SolrInputDocument solrInputDocument, DatasetVersion datasetVersion) { if (datasetVersion != null && datasetVersion.getTermsOfUseAndAccess() != null) { + //test to see if the terms of use are the default set in 5.10 - if so and there's no license then don't add license to solr doc. + //fixes 10513 + if (datasetVersionService.isVersionDefaultCustomTerms(datasetVersion)){ + return; + } + String licenseName = "Custom Terms"; - if(datasetVersion.getTermsOfUseAndAccess().getLicense() != null) { + if (datasetVersion.getTermsOfUseAndAccess().getLicense() != null) { licenseName = datasetVersion.getTermsOfUseAndAccess().getLicense().getName(); + } else if (datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null) { + // this fixes #10513 for datasets harvested in oai_dc - these + // have neither the license id, nor any actual custom terms + return; } solrInputDocument.addField(SearchFields.DATASET_LICENSE, licenseName); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 6c427672e6d..a0c32d5c8ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1816,5 +1816,13 @@ public static String getStorageDriver(DataFile dataFile) { String storageIdentifier = dataFile.getStorageIdentifier(); return storageIdentifier.substring(0, storageIdentifier.indexOf(DataAccess.SEPARATOR)); } - + + /** + * Replace spaces with "_" and remove invalid chars + * @param fileNameIn - Name before sanitization NOTE: not full path since this method removes '/' and '\' + * @return filename without spaces or invalid chars + */ + public static String sanitizeFileName(String fileNameIn) { + return fileNameIn == null ? null : fileNameIn.replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", ""); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index 9786fda4217..f1440cc3c02 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -15,6 +15,7 @@ import java.util.*; import java.nio.file.Files; +import java.nio.file.Paths; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import java.util.logging.Level; import java.util.logging.Logger; @@ -695,33 +696,42 @@ private boolean examineZipfile(FileInputStream zip_file_stream){ this.filesListInDir.clear(); this.filesizeHash.clear(); this.fileGroups.clear(); - - try{ + + try{ ZipInputStream zipStream = new ZipInputStream(zip_file_stream); ZipEntry entry; - + List hiddenDirectories = new ArrayList<>(); while((entry = zipStream.getNextEntry())!=null){ + String zentryFileName = entry.getName(); + boolean isDirectory = entry.isDirectory(); + + Boolean skip = isDirectory || this.isFileToSkip(zentryFileName); + + // check if path is hidden + if (isDirectory && Files.isHidden(Paths.get(zentryFileName))) { + hiddenDirectories.add(zentryFileName); + logger.fine("Ignoring files under hidden directory: " + zentryFileName); + } else { + // check if the path was already found to be hidden + for (String hidden : hiddenDirectories) { + if (zentryFileName.startsWith(hidden)) { + skip = true; + break; + } + } + } - String zentryFileName = entry.getName(); - //msg("zip entry: " + entry.getName()); - // Skip files or folders starting with __ - if (this.isFileToSkip(zentryFileName)){ - continue; - } - - if (entry.isDirectory()) { - //String dirpath = outputFolder + "/" + zentryFileName; - //createDirectory(dirpath); - continue; + if (skip) { + continue; } - + String unzipFileName = this.getFileBasename(zentryFileName); if (unzipFileName==null){ logger.warning("Zip Entry Basename is an empty string: " + zentryFileName); continue; } String unzipFolderName = this.getFolderName(zentryFileName); - + String unzipFilePath = unzipFileName; if (unzipFolderName != null) { unzipFilePath = unzipFolderName + "/" + unzipFileName; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 95f14b79ece..c72dfc1d127 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -640,18 +640,26 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); Set datasetFieldTypes = new TreeSet<>(metadataBlock.getDatasetFieldTypes()); + for (DatasetFieldType datasetFieldType : datasetFieldTypes) { - boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldType.getId()); - boolean displayCondition = !printOnlyDisplayedOnCreateDatasetFieldTypes || - datasetFieldType.isDisplayOnCreate() || - requiredInOwnerDataverse; + Long datasetFieldTypeId = datasetFieldType.getId(); + boolean requiredAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldTypeId); + boolean includedAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeIncludedAsInputLevel(datasetFieldTypeId); + boolean isNotInputLevelInOwnerDataverse = ownerDataverse != null && !ownerDataverse.isDatasetFieldTypeInInputLevels(datasetFieldTypeId); + + DatasetFieldType parentDatasetFieldType = datasetFieldType.getParentDatasetFieldType(); + boolean isRequired = parentDatasetFieldType == null ? datasetFieldType.isRequired() : parentDatasetFieldType.isRequired(); + + boolean displayCondition = printOnlyDisplayedOnCreateDatasetFieldTypes + ? (datasetFieldType.isDisplayOnCreate() || isRequired || requiredAsInputLevelInOwnerDataverse) + : ownerDataverse == null || includedAsInputLevelInOwnerDataverse || isNotInputLevelInOwnerDataverse; + if (displayCondition) { fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); } } jsonObjectBuilder.add("fields", fieldsBuilder); - return jsonObjectBuilder; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0325a47f626..b645276ceaf 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1445,6 +1445,8 @@ dataset.exportBtn.itemLabel.json=JSON dataset.exportBtn.itemLabel.oai_ore=OAI_ORE dataset.exportBtn.itemLabel.dataciteOpenAIRE=OpenAIRE dataset.exportBtn.itemLabel.html=DDI HTML Codebook +license.none.chosen=No license or custom terms chosen +license.none.chosen.description=No custom terms have been entered for this dataset license.custom=Custom Dataset Terms license.custom.description=Custom terms specific to this dataset metrics.title=Metrics diff --git a/src/main/resources/db/migration/V6.3.0.1.sql b/src/main/resources/db/migration/V6.3.0.1.sql new file mode 100644 index 00000000000..fd9cd823868 --- /dev/null +++ b/src/main/resources/db/migration/V6.3.0.1.sql @@ -0,0 +1,10 @@ +UPDATE termsofuseandaccess SET license_id = (SELECT license.id FROM license WHERE license.name = 'CC0 1.0'), termsofuse = NULL +WHERE termsofuse = 'This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver' + AND license_id IS null + AND confidentialitydeclaration IS null + AND specialpermissions IS null + AND restrictions IS null + AND citationrequirements IS null + AND depositorrequirements IS null + AND conditions IS null + AND disclaimer IS null; diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 6d5b0a5fe4f..42a4c3ec5f5 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -46,8 +46,8 @@

- - + +

diff --git a/src/main/webapp/datasetLicenseInfoFragment.xhtml b/src/main/webapp/datasetLicenseInfoFragment.xhtml index 257f6b3b12f..a1bd604aa6a 100644 --- a/src/main/webapp/datasetLicenseInfoFragment.xhtml +++ b/src/main/webapp/datasetLicenseInfoFragment.xhtml @@ -22,7 +22,7 @@ xmlns:jsf="http://xmlns.jcp.org/jsf"> target="_blank">#{DatasetUtil:getLicenseName(DatasetPage.workingVersion)} -

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml index 5948047d845..d53c4bf4709 100644 --- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml +++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml @@ -48,7 +48,7 @@ - diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java new file mode 100644 index 00000000000..97999af3244 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldTest.java @@ -0,0 +1,64 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; +import edu.harvard.iq.dataverse.mocks.MocksFactory; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; + + +class DatasetFieldTest { + @Test + void testCreateNewEmptyDatasetField_withEmptyTemplate() { + Template template = new Template(); + + DatasetField field = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), template); + assertTrue(field.getTemplate() == template); + assertTrue(template.getDatasetFields().isEmpty()); + } + + @Test + void testNotEqualDatasetFields() { + DatasetFieldType type1 = new DatasetFieldType("subject", FieldType.TEXT, false); + Template template1 = new Template(); + DatasetField field1 = DatasetField.createNewEmptyDatasetField(type1, template1); + field1.setId(MocksFactory.nextId()); + DatasetFieldType type2 = new DatasetFieldType("subject", FieldType.TEXT, false); + Template template2 = new Template(); + DatasetField field2 = DatasetField.createNewEmptyDatasetField(type2, template2); + field2.setId(MocksFactory.nextId()); + + assertNotEquals(field1, field2); + assertNotEquals(field1, template2); + } + + @Test + void testEqualDatasetFields() { + DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field1.setId(100L); + DatasetField field2 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + + // Fields are not equal before both have IDs set + assertNotEquals(field1, field2); + + field2.setId(100L); + + assertEquals(field1, field2); + } + + @Test + void testCopyDatasetFields() { + DatasetField field1 = DatasetField.createNewEmptyDatasetField(new DatasetFieldType("subject", FieldType.TEXT, false), new Template()); + field1.setId(100L); + DatasetField field2 = field1.copy(field1.getTemplate()); + + assertNull(field2.getId()); + // A copy of a field should not be equal + assertNotEquals(field1, field2); + + assertEquals(field2.getDatasetFieldType(), field1.getDatasetFieldType()); + } +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index b072a803aa4..79cc46cfa79 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -702,8 +702,10 @@ public void testListMetadataBlocks() { Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken); setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); - String[] testInputLevelNames = {"geographicCoverage", "country"}; - Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, apiToken); + String[] testInputLevelNames = {"geographicCoverage", "country", "city"}; + boolean[] testRequiredInputLevels = {false, true, false}; + boolean[] testIncludedInputLevels = {false, true, true}; + Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat().statusCode(OK.getStatusCode()); // Dataverse not found @@ -769,6 +771,21 @@ public void testListMetadataBlocks() { assertThat(expectedAllMetadataBlockDisplayNames, hasItemInArray(actualMetadataBlockDisplayName2)); assertThat(expectedAllMetadataBlockDisplayNames, hasItemInArray(actualMetadataBlockDisplayName3)); + // Check dataset fields for the updated input levels are retrieved + int geospatialMetadataBlockIndex = actualMetadataBlockDisplayName1.equals("Geospatial Metadata") ? 0 : actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 2; + + // Since the included property of geographicCoverage is set to false, we should retrieve the total number of fields minus one + listMetadataBlocksResponse.then().assertThat() + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(10)); + + String actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); + String actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); + String actualMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); + + assertNull(actualMetadataField1); + assertNotNull(actualMetadataField2); + assertNotNull(actualMetadataField3); + // Existent dataverse and onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken); listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -785,16 +802,18 @@ public void testListMetadataBlocks() { assertThat(expectedOnlyDisplayedOnCreateMetadataBlockDisplayNames, hasItemInArray(actualMetadataBlockDisplayName2)); // Check dataset fields for the updated input levels are retrieved - int geospatialMetadataBlockIndex = actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 0; + geospatialMetadataBlockIndex = actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 0; listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(2)); + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(1)); - String actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); - String actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); + actualMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); + actualMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); + actualMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); - assertNotNull(actualMetadataField1); + assertNull(actualMetadataField1); assertNotNull(actualMetadataField2); + assertNull(actualMetadataField3); // User has no permissions on the requested dataverse Response createSecondUserResponse = UtilIT.createRandomUser(); @@ -898,12 +917,16 @@ public void testUpdateInputLevels() { // Update valid input levels String[] testInputLevelNames = {"geographicCoverage", "country"}; - Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, apiToken); + boolean[] testRequiredInputLevels = {true, false}; + boolean[] testIncludedInputLevels = {true, false}; + Response updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); + String actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + int geographicCoverageInputLevelIndex = actualInputLevelName.equals("geographicCoverage") ? 0 : 1; updateDataverseInputLevelsResponse.then().assertThat() - .body("data.inputLevels[0].required", equalTo(true)) - .body("data.inputLevels[0].include", equalTo(true)) - .body("data.inputLevels[1].required", equalTo(true)) - .body("data.inputLevels[1].include", equalTo(true)) + .body(String.format("data.inputLevels[%d].include", geographicCoverageInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].required", geographicCoverageInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].include", 1 - geographicCoverageInputLevelIndex), equalTo(false)) + .body(String.format("data.inputLevels[%d].required", 1 - geographicCoverageInputLevelIndex), equalTo(false)) .statusCode(OK.getStatusCode()); String actualFieldTypeName1 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); String actualFieldTypeName2 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[1].datasetFieldTypeName"); @@ -913,15 +936,14 @@ public void testUpdateInputLevels() { // Update input levels with an invalid field type name String[] testInvalidInputLevelNames = {"geographicCoverage", "invalid1"}; - updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInvalidInputLevelNames, apiToken); + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInvalidInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat() .body("message", equalTo("Invalid dataset field type name: invalid1")) .statusCode(BAD_REQUEST.getStatusCode()); // Update invalid empty input levels testInputLevelNames = new String[]{}; - updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, apiToken); - updateDataverseInputLevelsResponse.prettyPrint(); + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, apiToken); updateDataverseInputLevelsResponse.then().assertThat() .body("message", equalTo("Error while updating dataverse input levels: Input level list cannot be null or empty")) .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java index 4df6c89411d..518431bfa2d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java @@ -462,7 +462,7 @@ public void testCreateAndDeleteDatasetInRoot() { assertNull(attemptToGetFileId); } catch (Exception ex) { System.out.println("We expect an exception here because we can no longer find the file because deleted it: " + ex); - assertTrue(ex.getClass().getName().equals(ArrayIndexOutOfBoundsException.class.getName())); + assertTrue(ex instanceof ArrayIndexOutOfBoundsException); } String newTitle = "A New Hope"; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 257610dbc32..0216859b869 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3960,22 +3960,25 @@ static Response requestGlobusUploadPaths(Integer datasetId, JsonObject body, Str .post("/api/datasets/" + datasetId + "/requestGlobusUploadPaths"); } - static Response updateDataverseInputLevels(String dataverseAlias, String[] inputLevelNames, String apiToken) { - JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); - for(String inputLevelName : inputLevelNames) { - contactArrayBuilder.add(Json.createObjectBuilder() - .add("datasetFieldTypeName", inputLevelName) - .add("required", true) - .add("include", true) - ); + public static Response updateDataverseInputLevels(String dataverseAlias, String[] inputLevelNames, boolean[] requiredInputLevels, boolean[] includedInputLevels, String apiToken) { + JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder(); + for (int i = 0; i < inputLevelNames.length; i++) { + inputLevelsArrayBuilder.add(createInputLevelObject(inputLevelNames[i], requiredInputLevels[i], includedInputLevels[i])); } return given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .body(contactArrayBuilder.build().toString()) + .body(inputLevelsArrayBuilder.build().toString()) .contentType(ContentType.JSON) .put("/api/dataverses/" + dataverseAlias + "/inputLevels"); } + private static JsonObjectBuilder createInputLevelObject(String name, boolean required, boolean include) { + return Json.createObjectBuilder() + .add("datasetFieldTypeName", name) + .add("required", required) + .add("include", include); + } + public static Response getOpenAPI(String accept, String format) { Response response = given() .header("Accept", accept) diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index 92b06e5936f..c062f63e264 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -51,6 +51,7 @@ public void setUp() { indexService.settingsService = Mockito.mock(SettingsServiceBean.class); indexService.dataverseService = Mockito.mock(DataverseServiceBean.class); indexService.datasetFieldService = Mockito.mock(DatasetFieldServiceBean.class); + indexService.datasetVersionService = Mockito.mock(DatasetVersionServiceBean.class); BrandingUtil.injectServices(indexService.dataverseService, indexService.settingsService); Mockito.when(indexService.dataverseService.findRootDataverse()).thenReturn(dataverse); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index ce8698c95eb..46359d7b02c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -434,4 +434,11 @@ public void testDetermineFileTypeROCrate() { assertEquals("Code", FileUtil.getIndexableFacetFileType(dockerDataFile)); } + @Test + public void testSanitizeFileName() { + assertEquals(null, FileUtil.sanitizeFileName(null)); + assertEquals("with_space", FileUtil.sanitizeFileName("with space")); + assertEquals("withcomma", FileUtil.sanitizeFileName("with,comma")); + assertEquals("with.txt", FileUtil.sanitizeFileName("with,\\?:;,.txt")); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java b/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java index f43a0c78284..9fac4d42bcd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/bagit/data/FileDataProviderFactoryTest.java @@ -23,21 +23,21 @@ public class FileDataProviderFactoryTest { public void should_return_FolderDataProvider_when_parameter_is_path() { FileDataProvider result = target.getFileDataProvider(Path.of(UUID.randomUUID().toString())); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(FolderDataProvider.class.getName())); + MatcherAssert.assertThat("should return FolderDataProvider when parameter is path", result instanceof FolderDataProvider); } @Test public void should_return_ZipFileDataProvider_when_parameter_is_file() throws IOException { FileDataProvider result = target.getFileDataProvider(Path.of(FIXTURE_DIRECTORY, "FileDataProviderFactoryTest.zip").toFile()); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(ZipFileDataProvider.class.getName())); + MatcherAssert.assertThat("should return ZipFileDataProvider when parameter is file", result instanceof ZipFileDataProvider); } @Test public void should_return_DataFileDataProvider_when_parameter_is_datafiles() { FileDataProvider result = target.getFileDataProvider("test-name", Collections.emptyList()); - MatcherAssert.assertThat(result.getClass().getName(), Matchers.is(DataFileDataProvider.class.getName())); + MatcherAssert.assertThat("should return DataFileDataProvider when parameter is datafiles", result instanceof DataFileDataProvider); } } \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java index f0e538616b2..3c5b4797b0a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java @@ -282,8 +282,14 @@ public void testZippedShapefileWithExtraFiles() throws IOException{ msg("Passed!"); } - + @Test + public void testHiddenFiles() { + // test with shapefiles in hidden directory + ShapefileHandler shp_handler = new ShapefileHandler("src/test/resources/hiddenShapefiles.zip"); + shp_handler.DEBUG= true; + assertFalse(shp_handler.containsShapefile()); + } diff --git a/src/test/resources/hiddenShapefiles.zip b/src/test/resources/hiddenShapefiles.zip new file mode 100644 index 00000000000..c733a2083ae Binary files /dev/null and b/src/test/resources/hiddenShapefiles.zip differ