Re-apply PATCH-4 to v6.0

doc/release-notes/9955-Signposting-updates.md

@@ -0,0 +1 @@
+This release fixes two issues (#9952, #9953) where the Signposting output did not match the Signposting specification.

doc/release-notes/9983-unique-constraints.md

@@ -0,0 +1,9 @@
+This release adds two missing database constraints that will assure that the externalvocabularyvalue table only has one entry for each uri and that the oaiset table only has one set for each spec. (In the very unlikely case that your existing database has duplicate entries now, install would fail. This can be checked by running
+
+SELECT uri, count(*) FROM externalvocabularyvaluet group by uri;
+
+and
+
+SELECT spec, count(*) FROM oaiset group by spec;
+
+and then removing any duplicate rows (where count>1).

doc/sphinx-guides/source/api/native-api.rst

@@ -2171,11 +2171,11 @@ Signposting involves the addition of a `Link <https://tools.ietf.org/html/rfc598
 
 Here is an example of a "Link" header:
 
-``Link: <https://doi.org/10.5072/FK2/YD5QDG>;rel="cite-as", <https://doi.org/10.5072/FK2/YD5QDG>;rel="describedby";type="application/vnd.citationstyles.csl+json",<https://demo.dataverse.org/api/datasets/export?exporter=schema.org&persistentId=doi:10.5072/FK2/YD5QDG>;rel="describedby";type="application/json+ld", <https://schema.org/AboutPage>;rel="type",<https://schema.org/Dataset>;rel="type", https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/customlicense?persistentId=doi:10.5072/FK2/YD5QDG;rel="license", <https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/linkset?persistentId=doi:10.5072/FK2/YD5QDG> ; rel="linkset";type="application/linkset+json"``
+``Link: <https://doi.org/10.5072/FK2/YD5QDG>;rel="cite-as", <https://doi.org/10.5072/FK2/YD5QDG>;rel="describedby";type="application/vnd.citationstyles.csl+json",<https://demo.dataverse.org/api/datasets/export?exporter=schema.org&persistentId=doi:10.5072/FK2/YD5QDG>;rel="describedby";type="application/ld+json", <https://schema.org/AboutPage>;rel="type",<https://schema.org/Dataset>;rel="type", <https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/customlicense?persistentId=doi:10.5072/FK2/YD5QDG>;rel="license", <https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/linkset?persistentId=doi:10.5072/FK2/YD5QDG> ; rel="linkset";type="application/linkset+json"``
 
 The URL for linkset information is discoverable under the ``rel="linkset";type="application/linkset+json`` entry in the "Link" header, such as in the example above.
 
-The reponse includes a JSON object conforming to the `Signposting <https://signposting.org>`__ specification.
+The reponse includes a JSON object conforming to the `Signposting <https://signposting.org>`__ specification. As part of this conformance, unlike most Dataverse API responses, the output is not wrapped in a ``{"status":"OK","data":{`` object.
 Signposting is not supported for draft dataset versions.
 
 .. code-block:: bash

src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java

@@ -19,6 +19,8 @@
 
 import jakarta.ejb.EJB;
 import jakarta.ejb.Stateless;
+import jakarta.ejb.TransactionAttribute;
+import jakarta.ejb.TransactionAttributeType;
 import jakarta.inject.Named;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
@@ -34,6 +36,7 @@
 import jakarta.persistence.NoResultException;
 import jakarta.persistence.NonUniqueResultException;
 import jakarta.persistence.PersistenceContext;
+import jakarta.persistence.PersistenceException;
 import jakarta.persistence.TypedQuery;
 
 import org.apache.commons.codec.digest.DigestUtils;
@@ -448,6 +451,7 @@ public JsonObject getExternalVocabularyValue(String termUri) {
      * @param cvocEntry - the configuration for the DatasetFieldType associated with this term 
      * @param term - the term uri as a string
      */
+    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
     public void registerExternalTerm(JsonObject cvocEntry, String term) {
         String retrievalUri = cvocEntry.getString("retrieval-uri");
         String prefix = cvocEntry.getString("prefix", null);
@@ -500,7 +504,8 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep
                         .setRetryHandler(new DefaultHttpRequestRetryHandler(3, false))
                         .build()) {
                     HttpGet httpGet = new HttpGet(retrievalUri);
-                    httpGet.addHeader("Accept", "application/json+ld, application/json");
+                    //application/json+ld is for backward compatibility
+                    httpGet.addHeader("Accept", "application/ld+json, application/json+ld, application/json");
 
                     HttpResponse response = httpClient.execute(httpGet);
                     String data = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
@@ -517,6 +522,8 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep
                             logger.fine("Wrote value for term: " + term);
                         } catch (JsonException je) {
                             logger.severe("Error retrieving: " + retrievalUri + " : " + je.getMessage());
+                        } catch (PersistenceException e) {
+                            logger.fine("Problem persisting: " + retrievalUri + " : " + e.getMessage());
                         }
                     } else {
                         logger.severe("Received response code : " + statusCode + " when retrieving " + retrievalUri

src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java

@@ -284,7 +284,7 @@ public void setDisplayOnCreate(boolean displayOnCreate) {
     }
 
     public boolean isControlledVocabulary() {
-        return controlledVocabularyValues != null && !controlledVocabularyValues.isEmpty();
+        return allowControlledVocabulary;
     }
 
     /**

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -2863,6 +2863,12 @@ public void sort() {
     public String refresh() {
         logger.fine("refreshing");
 
+        //In v5.14, versionId was null here. In 6.0, it appears not to be.
+        //This check is to handle the null if it reappears/occurs under other circumstances
+        if(versionId==null) {
+            logger.warning("versionId was null in refresh");
+            versionId = workingVersion.getId();
+        }
         //dataset = datasetService.find(dataset.getId());
         dataset = null;
         workingVersion = null; 
@@ -2872,10 +2878,9 @@ public String refresh() {
         DatasetVersionServiceBean.RetrieveDatasetVersionResponse retrieveDatasetVersionResponse = null;
 
         if (versionId != null) {
-            // versionId must have been set by now, in the init() method, 
-            // regardless of how the page was originally called - by the dataset
-            // database id, by the persistent identifier, or by the db id of
-            // the version. 
+            // versionId must have been set by now (see null check above), in the init()
+            // method, regardless of how the page was originally called - by the dataset
+            // database id, by the persistent identifier, or by the db id of the version.
             this.workingVersion = datasetVersionService.findDeep(versionId);
             dataset = workingVersion.getDataset();
         } 
@@ -6144,7 +6149,10 @@ public String getWebloaderUrlForDataset(Dataset d) {
     String signpostingLinkHeader = null;
 
     public String getSignpostingLinkHeader() {
-        if (!workingVersion.isReleased()) {
+        if ((workingVersion==null) || (!workingVersion.isReleased())) {
+            if(workingVersion==null) {
+                logger.warning("workingVersion was null in getSignpostingLinkHeader");
+            }
             return null;
         }
         if (signpostingLinkHeader == null) {

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -272,7 +272,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id"
 
     @GET
     @Path("/export")
-    @Produces({"application/xml", "application/json", "application/html" })
+    @Produces({"application/xml", "application/json", "application/html", "application/ld+json" })
     public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) {
 
         try {
@@ -574,24 +574,26 @@ public Response getVersionMetadataBlock(@Context ContainerRequestContext crc,
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/linkset")
-    public Response getLinkset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        if ( ":draft".equals(versionId) ) {
+    public Response getLinkset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId,
+            @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        if (":draft".equals(versionId)) {
             return badRequest("Signposting is not supported on the :draft version");
         }
-        User user = getRequestUser(crc);
-        return response(req -> {
+        DataverseRequest req = createDataverseRequest(getRequestUser(crc));
+        try {
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            return ok(Json.createObjectBuilder().add(
-                    "linkset",
-                    new SignpostingResources(
-                            systemConfig,
-                            dsv,
-                            JvmSettings.SIGNPOSTING_LEVEL1_AUTHOR_LIMIT.lookupOptional().orElse(""),
-                            JvmSettings.SIGNPOSTING_LEVEL1_ITEM_LIMIT.lookupOptional().orElse("")
-                    ).getJsonLinkset()
-            )
-            );
-        }, user);
+            return Response
+                    .ok(Json.createObjectBuilder()
+                            .add("linkset",
+                                    new SignpostingResources(systemConfig, dsv,
+                                            JvmSettings.SIGNPOSTING_LEVEL1_AUTHOR_LIMIT.lookupOptional().orElse(""),
+                                            JvmSettings.SIGNPOSTING_LEVEL1_ITEM_LIMIT.lookupOptional().orElse(""))
+                                                    .getJsonLinkset())
+                            .build())
+                    .type(MediaType.APPLICATION_JSON).build();
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
     }
 
     @GET

src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java

@@ -71,14 +71,14 @@ public String getLinks() {
 
         String describedby = "<" + ds.getGlobalId().asURL().toString() + ">;rel=\"describedby\"" + ";type=\"" + "application/vnd.citationstyles.csl+json\"";
         describedby += ",<" + systemConfig.getDataverseSiteUrl() + "/api/datasets/export?exporter=schema.org&persistentId="
-                + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() + ">;rel=\"describedby\"" + ";type=\"application/json+ld\"";
+                + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() + ">;rel=\"describedby\"" + ";type=\"application/ld+json\"";
         valueList.add(describedby);
 
         String type = "<https://schema.org/AboutPage>;rel=\"type\"";
         type = "<https://schema.org/AboutPage>;rel=\"type\",<" + defaultFileTypeValue + ">;rel=\"type\"";
         valueList.add(type);
 
-        String licenseString = DatasetUtil.getLicenseURI(workingDatasetVersion) + ";rel=\"license\"";
+        String licenseString = "<" + DatasetUtil.getLicenseURI(workingDatasetVersion) + ">;rel=\"license\"";
         valueList.add(licenseString);
 
         String linkset = "<" + systemConfig.getDataverseSiteUrl() + "/api/datasets/:persistentId/versions/"
@@ -116,7 +116,7 @@ public JsonArrayBuilder getJsonLinkset() {
                         systemConfig.getDataverseSiteUrl() + "/api/datasets/export?exporter=schema.org&persistentId=" + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier()
                 ).add(
                         "type",
-                        "application/json+ld"
+                        "application/ld+json"
                 )
         );
         JsonArrayBuilder linksetJsonObj = Json.createArrayBuilder();

src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql

@@ -0,0 +1,16 @@
+DO $$
+BEGIN
+
+  BEGIN
+    ALTER TABLE externalvocabularyvalue ADD CONSTRAINT externalvocabularvalue_uri_key UNIQUE(uri);
+  EXCEPTION
+    WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint externalvocabularvalue_uri_key already exists';
+  END;
+
+  BEGIN
+    ALTER TABLE oaiset ADD CONSTRAINT oaiset_spec_key UNIQUE(spec);
+  EXCEPTION
+    WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint oaiset_spec_key already exists';
+  END;
+
+END $$;

src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java

@@ -80,6 +80,7 @@ public void testSignposting() {
         assertTrue(linkHeader.contains(datasetPid));
         assertTrue(linkHeader.contains("cite-as"));
         assertTrue(linkHeader.contains("describedby"));
+        assertTrue(linkHeader.contains("<http://creativecommons.org/publicdomain/zero/1.0>;rel=\"license\""));
 
         Pattern pattern = Pattern.compile("<([^<]*)> ; rel=\"linkset\";type=\"application\\/linkset\\+json\"");
         Matcher matcher = pattern.matcher(linkHeader);
@@ -92,7 +93,7 @@ public void testSignposting() {
 
         String responseString = linksetResponse.getBody().asString();
 
-        JsonObject data = JsonUtil.getJsonObject(responseString).getJsonObject("data");
+        JsonObject data = JsonUtil.getJsonObject(responseString);
         JsonObject lso = data.getJsonArray("linkset").getJsonObject(0);
         System.out.println("Linkset: " + lso.toString());
 
@@ -101,6 +102,16 @@ public void testSignposting() {
         assertTrue(lso.getString("anchor").indexOf("/dataset.xhtml?persistentId=" + datasetPid) > 0);
         assertTrue(lso.containsKey("describedby"));
 
+        // Test export URL from link header
+        // regex inspired by https://stackoverflow.com/questions/68860255/how-to-match-the-closest-opening-and-closing-brackets
+        Pattern exporterPattern = Pattern.compile("[<\\[][^()\\[\\]]*?exporter=schema.org[^()\\[\\]]*[>\\]]");
+        Matcher exporterMatcher = exporterPattern.matcher(linkHeader);
+        exporterMatcher.find();
+
+        Response exportDataset = UtilIT.exportDataset(datasetPid, "schema.org");
+        exportDataset.prettyPrint();
+        exportDataset.then().assertThat().statusCode(OK.getStatusCode());
+
     }
 
 }