From f2430c57b367a39ead1f73cbbbd5dfb40d3ad648 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 21 Oct 2024 10:59:50 -0400 Subject: [PATCH 1/7] Add API endpoint for comparing Dataset Versions --- ...-add-api-for-comparing-dataset-versions.md | 15 +++ doc/sphinx-guides/source/api/native-api.rst | 19 +++ .../harvard/iq/dataverse/DatasetVersion.java | 4 + .../dataverse/DatasetVersionDifference.java | 123 +++++++++++++++--- .../harvard/iq/dataverse/api/Datasets.java | 17 +++ .../CuratePublishedDatasetVersionCommand.java | 2 +- .../DatasetVersionDifferenceTest.java | 90 +++++++++++++ .../harvard/iq/dataverse/api/DatasetsIT.java | 80 ++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 20 ++- 9 files changed, 349 insertions(+), 21 deletions(-) create mode 100644 doc/release-notes/10888-add-api-for-comparing-dataset-versions.md create mode 100644 src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java diff --git a/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md b/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md new file mode 100644 index 00000000000..902bf1a4d02 --- /dev/null +++ b/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md @@ -0,0 +1,15 @@ +The following API have been added: + +/api/datasets/{persistentId}/versions/{versionId0}/compare/{versionId1} + +This API lists the changes between 2 dataset versions. The Json response shows the changes per field within the Metadata block and the Terms Of Access. Also listed are the files that have been added or removed. Files that have been modified will also display the new file data plus the fields that have been modified. + +Old and New values are represented by "0" and "1" respectively. +```json +[ + "ModifiedFieldName" = { + "0" : "old value", + "1" : "new value" + } +] +``` diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index f8b8620f121..e3aae7a122a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1642,6 +1642,25 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/metadata/citation" +Compare Versions of a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Returns a list of fields that have changed between 2 Dataset versions within the Metadata and Terms of Access. Also includes the files that have been added or removed as well as files that have been modified. +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export ID=24 + export VERSION0=1.0 + export VERSION1=:draft + + curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION0/compare/$VERSION1" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/datasets/24/versions/:latest-published/compare/:draft" + Update Metadata For a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 0433c425fd2..2a870c81767 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -102,6 +102,10 @@ public int compare(DatasetVersion o1, DatasetVersion o2) { } } }; + public static final JsonObjectBuilder compareVersions(DatasetVersion originalVersion, DatasetVersion newVersion) { + DatasetVersionDifference diff = new DatasetVersionDifference(newVersion, originalVersion); + return diff.compareVersionsAsJson(); + } // TODO: Determine the UI implications of various version states //IMPORTANT: If you add a new value to this enum, you will also have to modify the diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index eca0c84ae84..dc0c342cb03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -5,19 +5,17 @@ import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.util.StringUtil; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.logging.Logger; +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; +import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObjectBuilder; import org.apache.commons.lang3.StringUtils; import edu.harvard.iq.dataverse.util.BundleUtil; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; + +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; /** * @@ -34,6 +32,7 @@ public final class DatasetVersionDifference { private List addedFiles = new ArrayList<>(); private List removedFiles = new ArrayList<>(); private List changedFileMetadata = new ArrayList<>(); + private Map>> changedFileMetadataDiff = new HashMap<>(); private List changedVariableMetadata = new ArrayList<>(); private List replacedFiles = new ArrayList<>(); private List changedTermsAccess = new ArrayList<>(); @@ -122,9 +121,12 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin for (FileMetadata fmdn : newVersion.getFileMetadatas()) { if (fmdo.getDataFile().equals(fmdn.getDataFile())) { deleted = false; - if (!compareFileMetadatas(fmdo, fmdn)) { + Map> fileMetadataDiff = compareFileMetadatas(fmdo, fmdn); + if (!fileMetadataDiff.isEmpty()) { changedFileMetadata.add(fmdo); changedFileMetadata.add(fmdn); + // TODO: find a better key for the map. needs to be something that doesn't change + changedFileMetadataDiff.put(fmdo, fileMetadataDiff); } if (!variableMetadataUtil.compareVariableMetadata(fmdo,fmdn) || !compareVarGroup(fmdo, fmdn)) { changedVariableMetadata.add(fmdo); @@ -551,25 +553,40 @@ private boolean compareVarGroup(FileMetadata fmdo, FileMetadata fmdn) { } } - public static boolean compareFileMetadatas(FileMetadata fmdo, FileMetadata fmdn) { - + public static Map> compareFileMetadatas(FileMetadata fmdo, FileMetadata fmdn) { + Map> fileMetadataChanged = new HashMap<>(); + boolean equals = true; if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getDescription()), StringUtil.nullToEmpty(fmdn.getDescription()))) { - return false; + equals = false; + fileMetadataChanged.put("Description", + List.of(StringUtil.nullToEmpty(fmdo.getDescription()), StringUtil.nullToEmpty(fmdn.getDescription()))); } if (!StringUtils.equals(fmdo.getCategoriesByName().toString(), fmdn.getCategoriesByName().toString())) { - return false; + equals = false; + fileMetadataChanged.put("Categories", + List.of(fmdo.getCategoriesByName().toString(), fmdn.getCategoriesByName().toString())); } if (!StringUtils.equals(fmdo.getLabel(), fmdn.getLabel())) { - return false; + equals = false; + fileMetadataChanged.put("Label", + List.of(fmdo.getLabel(), fmdn.getLabel())); } if (!StringUtils.equals(fmdo.getProvFreeForm(), fmdn.getProvFreeForm())) { - return false; + equals = false; + fileMetadataChanged.put("ProvFreeForm", + List.of(fmdo.getProvFreeForm(), fmdn.getProvFreeForm())); } - - return fmdo.isRestricted() == fmdn.isRestricted(); + + if (fmdo.isRestricted() != fmdn.isRestricted()) { + equals = false; + fileMetadataChanged.put("isRestricted", + List.of(String.valueOf(fmdo.isRestricted()), String.valueOf(fmdn.isRestricted()))); + } + + return fileMetadataChanged; } private void compareValues(DatasetField originalField, DatasetField newField, boolean compound) { @@ -1819,4 +1836,74 @@ private static boolean fieldsAreDifferent(DatasetField originalField, DatasetFie } return false; } + public JsonObjectBuilder compareVersionsAsJson() { + JsonObjectBuilder job = new NullSafeJsonBuilder(); + + JsonObjectBuilder jobMetadata = new NullSafeJsonBuilder(); + List> byBlock = getDetailDataByBlock(); + for (List l : byBlock) { + for (DatasetField[] dsfArray : l) { + JsonObjectBuilder jb = new NullSafeJsonBuilder(); + if (dsfArray[0].getDatasetFieldType().isPrimitive()) { + jb.add("0", dsfArray[0].getRawValue()); + } else { + jb.add("0", dsfArray[0].getCompoundRawValue()); + } + if (dsfArray[1].getDatasetFieldType().isPrimitive()) { + jb.add("1", dsfArray[1].getRawValue()); + } else { + jb.add("1", dsfArray[1].getCompoundRawValue()); + } + jobMetadata.add(dsfArray[0].getDatasetFieldType().getTitle(), jb); + } + } + if (!byBlock.isEmpty()) { + job.add("Metadata", jobMetadata); + } + + // Format added, removed, and modified files + JsonObjectBuilder jobFiles = new NullSafeJsonBuilder(); + if (!addedFiles.isEmpty()) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + addedFiles.forEach(f -> jab.add(json(f))); + jobFiles.add("added", jab); + } + if (!removedFiles.isEmpty()) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + removedFiles.forEach(f -> jab.add(json(f))); + jobFiles.add("removed", jab); + } + if (!changedFileMetadata.isEmpty()) { + JsonArrayBuilder jabDiffFiles = Json.createArrayBuilder(); + changedFileMetadataDiff.entrySet().forEach(entry -> { + JsonObjectBuilder jobDiffFiles = new NullSafeJsonBuilder(); + jobDiffFiles.add("fileMetadata", json(entry.getKey())); + entry.getValue().entrySet().forEach(e -> { + JsonObjectBuilder jobDiffField = new NullSafeJsonBuilder(); + jobDiffField.add("0",e.getValue().get(0)); + jobDiffField.add("1",e.getValue().get(1)); + jobDiffFiles.add(e.getKey(), jobDiffField); + }); + jabDiffFiles.add(jobDiffFiles); + }); + jobFiles.add("modified", jabDiffFiles); + } + if (!addedFiles.isEmpty() || !removedFiles.isEmpty() || !changedFileMetadata.isEmpty()) { + job.add("Files", jobFiles); + } + + // Format Terms Of Access changes + if (!changedTermsAccess.isEmpty()) { + JsonObjectBuilder jobTOA = new NullSafeJsonBuilder(); + changedTermsAccess.forEach(toa -> { + JsonObjectBuilder jobValue = new NullSafeJsonBuilder(); + jobValue.add("0",toa[1]); + jobValue.add("1",toa[2]); + jobTOA.add(toa[0], jobValue); + }); + job.add("TermsOfAccess", jobTOA); + } + + return job; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 369a22fe8d7..d7b0c78e611 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2992,6 +2992,23 @@ public Response cleanStorage(@Context ContainerRequestContext crc, @PathParam("i } + @GET + @AuthRequired + @Path("{id}/versions/{versionId1}/compare/{versionId2}") + public Response getCompareVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, + @PathParam("versionId1") String versionId1, + @PathParam("versionId2") String versionId2, + @Context UriInfo uriInfo, @Context HttpHeaders headers) { + try { + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + DatasetVersion dsv1 = getDatasetVersionOrDie(req, versionId1, findDatasetOrDie(id), uriInfo, headers); + DatasetVersion dsv2 = getDatasetVersionOrDie(req, versionId2, findDatasetOrDie(id), uriInfo, headers); + return ok(DatasetVersion.compareVersions(dsv1, dsv2)); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + private static Set getDatasetFilenames(Dataset dataset) { Set files = new HashSet<>(); for (DataFile dataFile: dataset.getFiles()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index e6e8279a314..e378e2e2ef7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -131,7 +131,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasetversion.update.failure"), this); } else { - metadataUpdated = DatasetVersionDifference.compareFileMetadatas(publishedFmd, draftFmd); + metadataUpdated = !DatasetVersionDifference.compareFileMetadatas(publishedFmd, draftFmd).isEmpty(); publishedFmd.setLabel(draftFmd.getLabel()); publishedFmd.setDescription(draftFmd.getDescription()); publishedFmd.setCategories(draftFmd.getCategories()); diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java new file mode 100644 index 00000000000..9457cdbdb4d --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -0,0 +1,90 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.util.json.JsonUtil; +import io.restassured.path.json.JsonPath; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.util.DateUtil.now; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DatasetVersionDifferenceTest { + + @Test + public void testCompareVersionsAsJson() { + + Dataverse dv = new Dataverse(); + Dataset ds = new Dataset(); + ds.setOwner(dv); + ds.setGlobalId(new GlobalId(AbstractDOIProvider.DOI_PROTOCOL,"10.5072","FK2/BYM3IW", "/", AbstractDOIProvider.DOI_RESOLVER_URL, null)); + + DatasetVersion dv1 = initDatasetVersion(0L, ds, DatasetVersion.VersionState.RELEASED); + DatasetVersion dv2 = initDatasetVersion(1L, ds, DatasetVersion.VersionState.DRAFT); + ds.setVersions(List.of(dv1, dv2)); + + TermsOfUseAndAccess toa = new TermsOfUseAndAccess(); + toa.setDisclaimer("disclaimer"); + dv2.setTermsOfUseAndAccess(toa); + dv2.getFileMetadatas().remove(1); + DatasetField dsf = new DatasetField(); + dsf.setDatasetFieldType(new DatasetFieldType("Author", DatasetFieldType.FieldType.TEXT, true)); + dsf.setSingleValue("TEST"); + dv2.getDatasetFields().add(dsf); + dv2.getFileMetadatas().get(2).setRestricted(!dv2.getFileMetadatas().get(2).isRestricted()); + DatasetVersionDifference dvd = new DatasetVersionDifference(dv2, dv1); + + JsonObjectBuilder json = dvd.compareVersionsAsJson(); + JsonObject obj = json.build(); + System.out.println(JsonUtil.prettyPrint(obj)); + + JsonPath dataFile = JsonPath.from(JsonUtil.prettyPrint(obj)); + assertTrue("TEST".equalsIgnoreCase(dataFile.getString("Metadata.Author.1"))); + assertTrue("true".equalsIgnoreCase(dataFile.getString("Files.modified[0].isRestricted.1"))); + assertTrue("disclaimer".equalsIgnoreCase(dataFile.getString("TermsOfAccess.Disclaimer.1"))); + } + private DatasetVersion initDatasetVersion(Long id, Dataset ds, DatasetVersion.VersionState vs) { + DatasetVersion dv = new DatasetVersion(); + dv.setDataset(ds); + dv.setVersion(1L); + dv.setVersionState(vs); + dv.setMinorVersionNumber(0L); + if (vs == DatasetVersion.VersionState.RELEASED) { + dv.setVersionNumber(1L); + dv.setVersion(1L); + dv.setReleaseTime(now()); + } + dv.setId(id); + dv.setTermsOfUseAndAccess(new TermsOfUseAndAccess()); + dv.setFileMetadatas(initFiles(dv)); + return dv; + } + private List initFiles(DatasetVersion dsv) { + List fileMetadata = new ArrayList<>(); + for (int i=0; i < 4; i++) { + FileMetadata fm = new FileMetadata(); + fm.setDatasetVersion(dsv); + DataFile df = new DataFile(); + DataTable dt = new DataTable(); + dt.setOriginalFileName("filename"+i+".txt"); + df.setId(Long.valueOf(i)); + df.setDescription("Desc"+i); + df.setRestricted(false); + df.setFilesize(100 + i); + df.setChecksumType(DataFile.ChecksumType.MD5); + df.setChecksumValue("value"+i); + df.setDataTable(dt); + df.setOwner(dsv.getDataset()); + fm.setDataFile(df); + fm.setLabel("Label"+i); + fileMetadata.add(fm); + df.setFileMetadatas(fileMetadata); + + } + return fileMetadata; + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 93f1024ae7a..af0f218bcf6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5168,4 +5168,84 @@ public void testGetCanDownloadAtLeastOneFile() { Response getUserPermissionsOnDatasetInvalidIdResponse = UtilIT.getCanDownloadAtLeastOneFile("testInvalidId", DS_VERSION_LATEST, secondUserApiToken); getUserPermissionsOnDatasetInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } + + @Test + public void testCompareDatasetVersionsAPI() { + + Response createUser = UtilIT.createRandomUser(); + assertEquals(200, createUser.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + Response getDatasetJsonBeforePublishing = UtilIT.nativeGet(datasetId, apiToken); + String protocol = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.protocol"); + String authority = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.authority"); + String identifier = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.identifier"); + String datasetPersistentId = protocol + ":" + authority + "/" + identifier; + + String pathToFile = "src/main/webapp/resources/images/dataverse-icon-1200.png"; + Response uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer modifyFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + pathToFile = "src/main/webapp/resources/images/dataverseproject_logo.jpg"; + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer deleteFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); + assertEquals(200, publishDataset.getStatusCode()); + + // post publish update to create DRAFT version + String pathToJsonFilePostPub = "doc/sphinx-guides/source/_static/api/dataset-add-metadata-after-pub.json"; + Response addDataToPublishedVersion = UtilIT.addDatasetMetadataViaNative(datasetPersistentId, pathToJsonFilePostPub, apiToken); + addDataToPublishedVersion.then().assertThat().statusCode(OK.getStatusCode()); + + // Test adding a file + pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + uploadResponse.prettyPrint(); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Test removing a file + uploadResponse = UtilIT.deleteFile(deleteFileId, apiToken); + uploadResponse.prettyPrint(); + uploadResponse.then().assertThat() + .statusCode(NO_CONTENT.getStatusCode()); + + // Test modify by restricting the file + Response restrictResponse = UtilIT.restrictFile(modifyFileId.toString(), true, apiToken); + restrictResponse.prettyPrint(); + restrictResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Also test a terms of access change + String jsonLDTerms = "{\"https://dataverse.org/schema/core#fileTermsOfAccess\":{\"https://dataverse.org/schema/core#dataAccessPlace\":\"Somewhere\"}}"; + Response updateTerms = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, jsonLDTerms, true); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response compareResponse = UtilIT.compareDatasetVersions(datasetPersistentId, ":latest-published", ":draft", apiToken); + compareResponse.prettyPrint(); + compareResponse.then().assertThat() + .body("data.Metadata.Author.1", CoreMatchers.containsString("Poe, Edgar Allen")) + .body("data.Files.added[0].label", CoreMatchers.equalTo("dataverseproject.png")) + .body("data.Files.removed[0].label", CoreMatchers.equalTo("dataverseproject_logo.jpg")) + .body("data.Files.modified[0].isRestricted.1", CoreMatchers.equalTo("true")) + .body("data.TermsOfAccess", CoreMatchers.notNullValue()) + .statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 70f49d81b35..26382884dd5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -323,7 +323,14 @@ static Integer getDatasetIdFromResponse(Response createDatasetResponse) { logger.info("Id found in create dataset response: " + datasetId); return datasetId; } - + + static Integer getDataFileIdFromResponse(Response uploadDataFileResponse) { + JsonPath dataFile = JsonPath.from(uploadDataFileResponse.body().asString()); + int dataFileId = dataFile.getInt("data.files[0].dataFile.id"); + logger.info("Id found in upload DataFile response: " + dataFileId); + return dataFileId; + } + static Integer getSearchCountFromResponse(Response searchResponse) { JsonPath createdDataset = JsonPath.from(searchResponse.body().asString()); int searchCount = createdDataset.getInt("data.total_count"); @@ -1570,7 +1577,16 @@ static Response getDatasetVersion(String persistentId, String versionNumber, Str + persistentId + (excludeFiles ? "&excludeFiles=true" : "")); } - + static Response compareDatasetVersions(String persistentId, String versionNumber1, String versionNumber2, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/:persistentId/versions/" + + versionNumber1 + + "/compare/" + + versionNumber2 + + "?persistentId=" + + persistentId); + } static Response getDatasetWithOwners(String persistentId, String apiToken, boolean returnOwners) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) From c12932f0ca46582fcd5f2674db0bb9b895cb0b13 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:42:07 -0400 Subject: [PATCH 2/7] updaing json format --- ...-add-api-for-comparing-dataset-versions.md | 18 ++-- .../dataverse/DatasetVersionDifference.java | 97 +++++++++++-------- .../DatasetVersionDifferenceTest.java | 10 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 16 ++- 4 files changed, 88 insertions(+), 53 deletions(-) diff --git a/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md b/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md index 902bf1a4d02..f9b3822d29d 100644 --- a/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md +++ b/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md @@ -4,12 +4,16 @@ The following API have been added: This API lists the changes between 2 dataset versions. The Json response shows the changes per field within the Metadata block and the Terms Of Access. Also listed are the files that have been added or removed. Files that have been modified will also display the new file data plus the fields that have been modified. -Old and New values are represented by "0" and "1" respectively. +Example of Metadata Block field change: ```json -[ - "ModifiedFieldName" = { - "0" : "old value", - "1" : "new value" - } -] +{ + "blockName": "Life Sciences Metadata", + "changed": [ + { + "fieldName": "Design Type", + "oldValue": "", + "newValue": "Parallel Group Design; Nested Case Control Design" + } + ] +} ``` diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index dc0c342cb03..66542b18c8e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.util.StringUtil; +import java.text.SimpleDateFormat; import java.util.*; import java.util.logging.Logger; @@ -15,8 +16,6 @@ import org.apache.commons.lang3.StringUtils; import edu.harvard.iq.dataverse.util.BundleUtil; -import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; - /** * * @author skraffmiller @@ -1838,69 +1837,89 @@ private static boolean fieldsAreDifferent(DatasetField originalField, DatasetFie } public JsonObjectBuilder compareVersionsAsJson() { JsonObjectBuilder job = new NullSafeJsonBuilder(); + JsonObjectBuilder jobVersion = new NullSafeJsonBuilder(); + jobVersion.add("versionNumber", originalVersion.getFriendlyVersionNumber()); + jobVersion.add("createdDate", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(originalVersion.getCreateTime())); + job.add("oldVersion", jobVersion); + jobVersion = new NullSafeJsonBuilder(); + jobVersion.add("versionNumber", newVersion.getFriendlyVersionNumber()); + jobVersion.add("createdDate", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(newVersion.getCreateTime())); + job.add("newVersion", jobVersion); - JsonObjectBuilder jobMetadata = new NullSafeJsonBuilder(); - List> byBlock = getDetailDataByBlock(); - for (List l : byBlock) { - for (DatasetField[] dsfArray : l) { - JsonObjectBuilder jb = new NullSafeJsonBuilder(); - if (dsfArray[0].getDatasetFieldType().isPrimitive()) { - jb.add("0", dsfArray[0].getRawValue()); - } else { - jb.add("0", dsfArray[0].getCompoundRawValue()); - } - if (dsfArray[1].getDatasetFieldType().isPrimitive()) { - jb.add("1", dsfArray[1].getRawValue()); - } else { - jb.add("1", dsfArray[1].getCompoundRawValue()); + if (!this.detailDataByBlock.isEmpty()) { + JsonArrayBuilder jabMetadata = Json.createArrayBuilder(); + for (List blocks : detailDataByBlock) { + JsonObjectBuilder jobMetadata = new NullSafeJsonBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + String blockDisplay = blocks.get(0)[0].getDatasetFieldType().getMetadataBlock().getDisplayName(); + for (DatasetField[] dsfArray : blocks) { + JsonObjectBuilder jb = new NullSafeJsonBuilder(); + jb.add("fieldName", dsfArray[0].getDatasetFieldType().getTitle()); + if (dsfArray[0].getDatasetFieldType().isPrimitive()) { + jb.add("oldValue", dsfArray[0].getRawValue()); + } else { + jb.add("oldValue", dsfArray[0].getCompoundRawValue()); + } + if (dsfArray[1].getDatasetFieldType().isPrimitive()) { + jb.add("newValue", dsfArray[1].getRawValue()); + } else { + jb.add("newValue", dsfArray[1].getCompoundRawValue()); + } + jab.add(jb); } - jobMetadata.add(dsfArray[0].getDatasetFieldType().getTitle(), jb); + jobMetadata.add("blockName", blockDisplay); + jobMetadata.add("changed", jab); + jabMetadata.add(jobMetadata); } - } - if (!byBlock.isEmpty()) { - job.add("Metadata", jobMetadata); + job.add("metadataChanges", jabMetadata); } // Format added, removed, and modified files - JsonObjectBuilder jobFiles = new NullSafeJsonBuilder(); + JsonArrayBuilder jabDiffFiles = Json.createArrayBuilder(); if (!addedFiles.isEmpty()) { JsonArrayBuilder jab = Json.createArrayBuilder(); - addedFiles.forEach(f -> jab.add(json(f))); - jobFiles.add("added", jab); + addedFiles.forEach(f -> { + jab.add(new NullSafeJsonBuilder().add("fileName", f.getDataFile().getDisplayName())); + }); + job.add("filesAdded", jab); } if (!removedFiles.isEmpty()) { JsonArrayBuilder jab = Json.createArrayBuilder(); - removedFiles.forEach(f -> jab.add(json(f))); - jobFiles.add("removed", jab); + removedFiles.forEach(f -> { + jab.add(new NullSafeJsonBuilder().add("fileName", f.getDataFile().getDisplayName())); + }); + job.add("filesRemoved", jab); } if (!changedFileMetadata.isEmpty()) { - JsonArrayBuilder jabDiffFiles = Json.createArrayBuilder(); changedFileMetadataDiff.entrySet().forEach(entry -> { - JsonObjectBuilder jobDiffFiles = new NullSafeJsonBuilder(); - jobDiffFiles.add("fileMetadata", json(entry.getKey())); + JsonArrayBuilder jab = Json.createArrayBuilder(); + JsonObjectBuilder jobChanges = new NullSafeJsonBuilder(); + jobChanges.add("fileName", entry.getKey().getDataFile().getOriginalFileName()); entry.getValue().entrySet().forEach(e -> { JsonObjectBuilder jobDiffField = new NullSafeJsonBuilder(); - jobDiffField.add("0",e.getValue().get(0)); - jobDiffField.add("1",e.getValue().get(1)); - jobDiffFiles.add(e.getKey(), jobDiffField); + jobDiffField.add("fieldName",e.getKey()); + jobDiffField.add("oldValue",e.getValue().get(0)); + jobDiffField.add("newValue",e.getValue().get(1)); + jab.add(jobDiffField); }); - jabDiffFiles.add(jobDiffFiles); + jobChanges.add("changes", jab); + jabDiffFiles.add(jobChanges); }); - jobFiles.add("modified", jabDiffFiles); - } - if (!addedFiles.isEmpty() || !removedFiles.isEmpty() || !changedFileMetadata.isEmpty()) { - job.add("Files", jobFiles); + job.add("fileChanges", jabDiffFiles); } // Format Terms Of Access changes if (!changedTermsAccess.isEmpty()) { JsonObjectBuilder jobTOA = new NullSafeJsonBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); changedTermsAccess.forEach(toa -> { JsonObjectBuilder jobValue = new NullSafeJsonBuilder(); - jobValue.add("0",toa[1]); - jobValue.add("1",toa[2]); - jobTOA.add(toa[0], jobValue); + jobValue.add("fieldName",toa[0]); + jobValue.add("oldValue",toa[1]); + jobValue.add("newValue",toa[2]); + jab.add(jobValue); }); + jobTOA.add("changed", jab); job.add("TermsOfAccess", jobTOA); } diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java index 9457cdbdb4d..4b901f99afe 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -33,6 +33,9 @@ public void testCompareVersionsAsJson() { dv2.getFileMetadatas().remove(1); DatasetField dsf = new DatasetField(); dsf.setDatasetFieldType(new DatasetFieldType("Author", DatasetFieldType.FieldType.TEXT, true)); + MetadataBlock mb = new MetadataBlock(); + mb.setDisplayName("testMetadataBlock"); + dsf.getDatasetFieldType().setMetadataBlock(mb); dsf.setSingleValue("TEST"); dv2.getDatasetFields().add(dsf); dv2.getFileMetadatas().get(2).setRestricted(!dv2.getFileMetadatas().get(2).isRestricted()); @@ -43,9 +46,9 @@ public void testCompareVersionsAsJson() { System.out.println(JsonUtil.prettyPrint(obj)); JsonPath dataFile = JsonPath.from(JsonUtil.prettyPrint(obj)); - assertTrue("TEST".equalsIgnoreCase(dataFile.getString("Metadata.Author.1"))); - assertTrue("true".equalsIgnoreCase(dataFile.getString("Files.modified[0].isRestricted.1"))); - assertTrue("disclaimer".equalsIgnoreCase(dataFile.getString("TermsOfAccess.Disclaimer.1"))); + assertTrue("TEST".equalsIgnoreCase(dataFile.getString("metadataChanges[0].changed[0].newValue"))); + assertTrue("true".equalsIgnoreCase(dataFile.getString("fileChanges[0].changes[0].newValue"))); + assertTrue("disclaimer".equalsIgnoreCase(dataFile.getString("TermsOfAccess.changed[0].newValue"))); } private DatasetVersion initDatasetVersion(Long id, Dataset ds, DatasetVersion.VersionState vs) { DatasetVersion dv = new DatasetVersion(); @@ -59,6 +62,7 @@ private DatasetVersion initDatasetVersion(Long id, Dataset ds, DatasetVersion.Ve dv.setReleaseTime(now()); } dv.setId(id); + dv.setCreateTime(now()); dv.setTermsOfUseAndAccess(new TermsOfUseAndAccess()); dv.setFileMetadatas(initFiles(dv)); return dv; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index af0f218bcf6..4b022b00cef 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5241,10 +5241,18 @@ public void testCompareDatasetVersionsAPI() { Response compareResponse = UtilIT.compareDatasetVersions(datasetPersistentId, ":latest-published", ":draft", apiToken); compareResponse.prettyPrint(); compareResponse.then().assertThat() - .body("data.Metadata.Author.1", CoreMatchers.containsString("Poe, Edgar Allen")) - .body("data.Files.added[0].label", CoreMatchers.equalTo("dataverseproject.png")) - .body("data.Files.removed[0].label", CoreMatchers.equalTo("dataverseproject_logo.jpg")) - .body("data.Files.modified[0].isRestricted.1", CoreMatchers.equalTo("true")) + .body("data.oldVersion.versionNumber", CoreMatchers.equalTo("1.0")) + .body("data.newVersion.versionNumber", CoreMatchers.equalTo("DRAFT")) + .body("data.metadataChanges[0].blockName", CoreMatchers.equalTo("Citation Metadata")) + .body("data.metadataChanges[0].changed[0].fieldName", CoreMatchers.equalTo("Author")) + .body("data.metadataChanges[0].changed[0].oldValue", CoreMatchers.containsString("Finch, Fiona; (Birds Inc.)")) + .body("data.metadataChanges[1].blockName", CoreMatchers.equalTo("Life Sciences Metadata")) + .body("data.metadataChanges[1].changed[0].fieldName", CoreMatchers.equalTo("Design Type")) + .body("data.metadataChanges[1].changed[0].oldValue", CoreMatchers.containsString("")) + .body("data.metadataChanges[1].changed[0].newValue", CoreMatchers.containsString("Parallel Group Design; Nested Case Control Design")) + .body("data.filesAdded[0].fileName", CoreMatchers.equalTo("dataverseproject.png")) + .body("data.filesRemoved[0].fileName", CoreMatchers.equalTo("dataverseproject_logo.jpg")) + .body("data.fileChanges[0].changes[0].newValue", CoreMatchers.equalTo("true")) .body("data.TermsOfAccess", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); } From 842f04b7167cb9f1ee16226b40859f9955dd82b6 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:53:58 -0400 Subject: [PATCH 3/7] fixes for json output --- .../harvard/iq/dataverse/DatasetVersionDifference.java | 8 ++++---- .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 66542b18c8e..7e5750959ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -1893,8 +1893,8 @@ public JsonObjectBuilder compareVersionsAsJson() { if (!changedFileMetadata.isEmpty()) { changedFileMetadataDiff.entrySet().forEach(entry -> { JsonArrayBuilder jab = Json.createArrayBuilder(); - JsonObjectBuilder jobChanges = new NullSafeJsonBuilder(); - jobChanges.add("fileName", entry.getKey().getDataFile().getOriginalFileName()); + JsonObjectBuilder jobChanged = new NullSafeJsonBuilder(); + jobChanged.add("fileName", entry.getKey().getDataFile().getDisplayName()); entry.getValue().entrySet().forEach(e -> { JsonObjectBuilder jobDiffField = new NullSafeJsonBuilder(); jobDiffField.add("fieldName",e.getKey()); @@ -1902,8 +1902,8 @@ public JsonObjectBuilder compareVersionsAsJson() { jobDiffField.add("newValue",e.getValue().get(1)); jab.add(jobDiffField); }); - jobChanges.add("changes", jab); - jabDiffFiles.add(jobChanges); + jobChanged.add("changed", jab); + jabDiffFiles.add(jobChanged); }); job.add("fileChanges", jabDiffFiles); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4b022b00cef..4d515a64cf5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5252,7 +5252,8 @@ public void testCompareDatasetVersionsAPI() { .body("data.metadataChanges[1].changed[0].newValue", CoreMatchers.containsString("Parallel Group Design; Nested Case Control Design")) .body("data.filesAdded[0].fileName", CoreMatchers.equalTo("dataverseproject.png")) .body("data.filesRemoved[0].fileName", CoreMatchers.equalTo("dataverseproject_logo.jpg")) - .body("data.fileChanges[0].changes[0].newValue", CoreMatchers.equalTo("true")) + .body("data.fileChanges[0].fileName", CoreMatchers.equalTo("dataverse-icon-1200.png")) + .body("data.fileChanges[0].changed[0].newValue", CoreMatchers.equalTo("true")) .body("data.TermsOfAccess", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); } From c5adf921202991fafc222325d3dffa1b803525ee Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:18:52 -0400 Subject: [PATCH 4/7] fixes for json output --- .../dataverse/DatasetVersionDifference.java | 37 +++++++++- .../DatasetVersionDifferenceTest.java | 73 +++++++++++++------ .../harvard/iq/dataverse/api/DatasetsIT.java | 37 ++++++++-- 3 files changed, 114 insertions(+), 33 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 7e5750959ea..5805359feb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -1839,7 +1839,7 @@ public JsonObjectBuilder compareVersionsAsJson() { JsonObjectBuilder job = new NullSafeJsonBuilder(); JsonObjectBuilder jobVersion = new NullSafeJsonBuilder(); jobVersion.add("versionNumber", originalVersion.getFriendlyVersionNumber()); - jobVersion.add("createdDate", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(originalVersion.getCreateTime())); + jobVersion.add("lastUpdatedDate", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(originalVersion.getLastUpdateTime())); job.add("oldVersion", jobVersion); jobVersion = new NullSafeJsonBuilder(); jobVersion.add("versionNumber", newVersion.getFriendlyVersionNumber()); @@ -1879,22 +1879,36 @@ public JsonObjectBuilder compareVersionsAsJson() { if (!addedFiles.isEmpty()) { JsonArrayBuilder jab = Json.createArrayBuilder(); addedFiles.forEach(f -> { - jab.add(new NullSafeJsonBuilder().add("fileName", f.getDataFile().getDisplayName())); + jab.add(filesDiffJson(f)); }); job.add("filesAdded", jab); } if (!removedFiles.isEmpty()) { JsonArrayBuilder jab = Json.createArrayBuilder(); removedFiles.forEach(f -> { - jab.add(new NullSafeJsonBuilder().add("fileName", f.getDataFile().getDisplayName())); + jab.add(filesDiffJson(f)); }); job.add("filesRemoved", jab); } + if (!replacedFiles.isEmpty()) { + JsonArrayBuilder jabReplaced = Json.createArrayBuilder(); + replacedFiles.forEach(fm -> { + if (fm.length == 2) { + JsonObjectBuilder jobReplaced = new NullSafeJsonBuilder(); + jobReplaced.add("oldFile", filesDiffJson(fm[0])); + jobReplaced.add("newFile", filesDiffJson(fm[1])); + jabReplaced.add(jobReplaced); + } + }); + job.add("filesReplaced", jabReplaced); + } if (!changedFileMetadata.isEmpty()) { changedFileMetadataDiff.entrySet().forEach(entry -> { JsonArrayBuilder jab = Json.createArrayBuilder(); JsonObjectBuilder jobChanged = new NullSafeJsonBuilder(); jobChanged.add("fileName", entry.getKey().getDataFile().getDisplayName()); + jobChanged.add(entry.getKey().getDataFile().getChecksumType().name(), entry.getKey().getDataFile().getChecksumValue()); + jobChanged.add("fileId", entry.getKey().getDataFile().getId()); entry.getValue().entrySet().forEach(e -> { JsonObjectBuilder jobDiffField = new NullSafeJsonBuilder(); jobDiffField.add("fieldName",e.getKey()); @@ -1925,4 +1939,21 @@ public JsonObjectBuilder compareVersionsAsJson() { return job; } + private JsonObjectBuilder filesDiffJson(FileMetadata fileMetadata) { + NullSafeJsonBuilder job = new NullSafeJsonBuilder(); + DataFile df = fileMetadata.getDataFile(); + List tags = df.getTags(); + job.add("fileName", df.getDisplayName()) + .add(df.getChecksumType().name(), df.getChecksumValue()) + .add("type",df.getContentType()) + .add("fileId", df.getId()) + .add("description", df.getDescription()) + .add("isRestricted", df.isRestricted()); + if (df.getTags() != null && !df.getTags().isEmpty()) { + JsonArrayBuilder jabTags = Json.createArrayBuilder(); + df.getTags().forEach(t -> jabTags.add(t.getTypeLabel())); + job.add("tags", jabTags); + } + return job; + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java index 4b901f99afe..f65f510090e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -15,6 +15,7 @@ public class DatasetVersionDifferenceTest { + static Long fileId = Long.valueOf(0); @Test public void testCompareVersionsAsJson() { @@ -30,7 +31,6 @@ public void testCompareVersionsAsJson() { TermsOfUseAndAccess toa = new TermsOfUseAndAccess(); toa.setDisclaimer("disclaimer"); dv2.setTermsOfUseAndAccess(toa); - dv2.getFileMetadatas().remove(1); DatasetField dsf = new DatasetField(); dsf.setDatasetFieldType(new DatasetFieldType("Author", DatasetFieldType.FieldType.TEXT, true)); MetadataBlock mb = new MetadataBlock(); @@ -38,7 +38,16 @@ public void testCompareVersionsAsJson() { dsf.getDatasetFieldType().setMetadataBlock(mb); dsf.setSingleValue("TEST"); dv2.getDatasetFields().add(dsf); - dv2.getFileMetadatas().get(2).setRestricted(!dv2.getFileMetadatas().get(2).isRestricted()); + // modify file at index 0 + dv2.getFileMetadatas().get(0).setRestricted(!dv2.getFileMetadatas().get(2).isRestricted()); + + FileMetadata addedFile = initFile(dv2); // add a new file + FileMetadata removedFile = dv2.getFileMetadatas().get(1); // remove the second file + dv2.getFileMetadatas().remove(1); + FileMetadata replacedFile = dv2.getFileMetadatas().get(1); // the third file is now at index 1 since the second file was removed + FileMetadata replacementFile = initFile(dv2, replacedFile.getDataFile().getId()); // replace the third file with a new file + dv2.getFileMetadatas().remove(1); + DatasetVersionDifference dvd = new DatasetVersionDifference(dv2, dv1); JsonObjectBuilder json = dvd.compareVersionsAsJson(); @@ -47,7 +56,11 @@ public void testCompareVersionsAsJson() { JsonPath dataFile = JsonPath.from(JsonUtil.prettyPrint(obj)); assertTrue("TEST".equalsIgnoreCase(dataFile.getString("metadataChanges[0].changed[0].newValue"))); - assertTrue("true".equalsIgnoreCase(dataFile.getString("fileChanges[0].changes[0].newValue"))); + assertTrue(addedFile.getLabel().equalsIgnoreCase(dataFile.getString("filesAdded[0].fileName"))); + assertTrue(removedFile.getLabel().equalsIgnoreCase(dataFile.getString("filesRemoved[0].fileName"))); + assertTrue(replacedFile.getLabel().equalsIgnoreCase(dataFile.getString("filesReplaced[0].oldFile.fileName"))); + assertTrue(replacementFile.getLabel().equalsIgnoreCase(dataFile.getString("filesReplaced[0].newFile.fileName"))); + assertTrue("true".equalsIgnoreCase(dataFile.getString("fileChanges[0].changed[0].newValue"))); assertTrue("disclaimer".equalsIgnoreCase(dataFile.getString("TermsOfAccess.changed[0].newValue"))); } private DatasetVersion initDatasetVersion(Long id, Dataset ds, DatasetVersion.VersionState vs) { @@ -63,32 +76,44 @@ private DatasetVersion initDatasetVersion(Long id, Dataset ds, DatasetVersion.Ve } dv.setId(id); dv.setCreateTime(now()); + dv.setLastUpdateTime(now()); dv.setTermsOfUseAndAccess(new TermsOfUseAndAccess()); dv.setFileMetadatas(initFiles(dv)); return dv; } private List initFiles(DatasetVersion dsv) { - List fileMetadata = new ArrayList<>(); - for (int i=0; i < 4; i++) { - FileMetadata fm = new FileMetadata(); - fm.setDatasetVersion(dsv); - DataFile df = new DataFile(); - DataTable dt = new DataTable(); - dt.setOriginalFileName("filename"+i+".txt"); - df.setId(Long.valueOf(i)); - df.setDescription("Desc"+i); - df.setRestricted(false); - df.setFilesize(100 + i); - df.setChecksumType(DataFile.ChecksumType.MD5); - df.setChecksumValue("value"+i); - df.setDataTable(dt); - df.setOwner(dsv.getDataset()); - fm.setDataFile(df); - fm.setLabel("Label"+i); - fileMetadata.add(fm); - df.setFileMetadatas(fileMetadata); - + List fileMetadatas = new ArrayList<>(); + fileId = 0L; + for (int i=0; i < 10; i++) { + FileMetadata fm = initFile(dsv); + fileMetadatas.add(fm); } - return fileMetadata; + return fileMetadatas; + } + private FileMetadata initFile(DatasetVersion dsv) { + return initFile(dsv, null); + } + private FileMetadata initFile(DatasetVersion dsv, Long prevId) { + Long id = fileId++; + FileMetadata fm = new FileMetadata(); + DataFile df = new DataFile(); + fm.setDatasetVersion(dsv); + DataTable dt = new DataTable(); + dt.setOriginalFileName("filename"+id+".txt"); + df.setId(id); + df.setDescription("Desc"+id); + df.setRestricted(false); + df.setFilesize(100 + id); + df.setChecksumType(DataFile.ChecksumType.MD5); + df.setChecksumValue("value"+id); + df.setDataTable(dt); + df.setOwner(dsv.getDataset()); + df.getFileMetadatas().add(fm); + df.setPreviousDataFileId(prevId); + fm.setId(id); + fm.setDataFile(df); + fm.setLabel("Label"+id); + dsv.getFileMetadatas().add(fm); + return fm; } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4d515a64cf5..bf292723ba5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5170,7 +5170,7 @@ public void testGetCanDownloadAtLeastOneFile() { } @Test - public void testCompareDatasetVersionsAPI() { + public void testCompareDatasetVersionsAPI() throws InterruptedException { Response createUser = UtilIT.createRandomUser(); assertEquals(200, createUser.getStatusCode()); @@ -5202,6 +5202,12 @@ public void testCompareDatasetVersionsAPI() { .statusCode(OK.getStatusCode()); Integer deleteFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + pathToFile = "src/main/webapp/resources/images/fav/favicon-16x16.png"; + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + uploadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer replaceFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); assertEquals(200, publishDataverse.getStatusCode()); @@ -5214,10 +5220,20 @@ public void testCompareDatasetVersionsAPI() { addDataToPublishedVersion.then().assertThat().statusCode(OK.getStatusCode()); // Test adding a file - pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; - uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); - uploadResponse.prettyPrint(); - uploadResponse.then().assertThat() + pathToFile = "src/test/resources/tab/test.tab"; + Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile, Json.createObjectBuilder().build(), apiToken); + uploadTabularFileResponse.prettyPrint(); + uploadTabularFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Integer addedFileId = UtilIT.getDataFileIdFromResponse(uploadTabularFileResponse); + + // Ensure tabular file is ingested + sleep(2000); + + String tabularTagName = "Survey"; + Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(String.valueOf(addedFileId), apiToken, List.of(tabularTagName)); + setFileTabularTagsResponse.prettyPrint(); + setFileTabularTagsResponse.then().assertThat() .statusCode(OK.getStatusCode()); // Test removing a file @@ -5226,6 +5242,12 @@ public void testCompareDatasetVersionsAPI() { uploadResponse.then().assertThat() .statusCode(NO_CONTENT.getStatusCode()); + // Test Replacing a file + Response replaceResponse = UtilIT.replaceFile(String.valueOf(replaceFileId), "src/main/webapp/resources/images/fav/favicon-32x32.png", apiToken); + replaceResponse.prettyPrint(); + replaceResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + // Test modify by restricting the file Response restrictResponse = UtilIT.restrictFile(modifyFileId.toString(), true, apiToken); restrictResponse.prettyPrint(); @@ -5250,10 +5272,13 @@ public void testCompareDatasetVersionsAPI() { .body("data.metadataChanges[1].changed[0].fieldName", CoreMatchers.equalTo("Design Type")) .body("data.metadataChanges[1].changed[0].oldValue", CoreMatchers.containsString("")) .body("data.metadataChanges[1].changed[0].newValue", CoreMatchers.containsString("Parallel Group Design; Nested Case Control Design")) - .body("data.filesAdded[0].fileName", CoreMatchers.equalTo("dataverseproject.png")) + .body("data.filesAdded[0].fileName", CoreMatchers.equalTo("test.tab")) + .body("data.filesAdded[0].tags[0]", CoreMatchers.equalTo("Survey")) .body("data.filesRemoved[0].fileName", CoreMatchers.equalTo("dataverseproject_logo.jpg")) .body("data.fileChanges[0].fileName", CoreMatchers.equalTo("dataverse-icon-1200.png")) .body("data.fileChanges[0].changed[0].newValue", CoreMatchers.equalTo("true")) + .body("data.filesReplaced[0].oldFile.fileName", CoreMatchers.equalTo("favicon-16x16.png")) + .body("data.filesReplaced[0].newFile.fileName", CoreMatchers.equalTo("favicon-32x32.png")) .body("data.TermsOfAccess", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); } From d957f60d5969c42697856adda927be160f0991c8 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:28:45 -0400 Subject: [PATCH 5/7] fixes for json output --- .../dataverse/DatasetVersionDifference.java | 11 +++++++--- .../iq/dataverse/util/json/JsonParser.java | 1 - .../DatasetVersionDifferenceTest.java | 2 ++ .../harvard/iq/dataverse/api/DatasetsIT.java | 22 ++++++++++++++----- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 5805359feb0..5760aa2c5b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -1843,7 +1843,7 @@ public JsonObjectBuilder compareVersionsAsJson() { job.add("oldVersion", jobVersion); jobVersion = new NullSafeJsonBuilder(); jobVersion.add("versionNumber", newVersion.getFriendlyVersionNumber()); - jobVersion.add("createdDate", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(newVersion.getCreateTime())); + jobVersion.add("lastUpdatedDate", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(newVersion.getLastUpdateTime())); job.add("newVersion", jobVersion); if (!this.detailDataByBlock.isEmpty()) { @@ -1942,13 +1942,18 @@ public JsonObjectBuilder compareVersionsAsJson() { private JsonObjectBuilder filesDiffJson(FileMetadata fileMetadata) { NullSafeJsonBuilder job = new NullSafeJsonBuilder(); DataFile df = fileMetadata.getDataFile(); - List tags = df.getTags(); job.add("fileName", df.getDisplayName()) + .add("filePath", fileMetadata.getDirectoryLabel()) .add(df.getChecksumType().name(), df.getChecksumValue()) .add("type",df.getContentType()) .add("fileId", df.getId()) - .add("description", df.getDescription()) + .add("description", fileMetadata.getDescription()) .add("isRestricted", df.isRestricted()); + if (fileMetadata.getCategories() != null && !fileMetadata.getCategories().isEmpty()) { + JsonArrayBuilder jabCategories = Json.createArrayBuilder(); + fileMetadata.getCategories().forEach(c -> jabCategories.add(c.getName())); + job.add("categories", jabCategories); + } if (df.getTags() != null && !df.getTags().isEmpty()) { JsonArrayBuilder jabTags = Json.createArrayBuilder(); df.getTags().forEach(t -> jabTags.add(t.getTypeLabel())); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 2f01c9bc2f2..65dd2986a86 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -965,7 +965,6 @@ public void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cv if (cvv == null) { if (allowHarvestingMissingCVV) { // we need to process this as a primitive value - logger.warning(">>>> Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'. Processing as primitive per setting override."); parsePrimitiveValue(dsf, cvvType , json); return; } else { diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java index f65f510090e..138ef709d18 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -113,6 +113,8 @@ private FileMetadata initFile(DatasetVersion dsv, Long prevId) { fm.setId(id); fm.setDataFile(df); fm.setLabel("Label"+id); + fm.setDirectoryLabel("/myFilePath/"); + fm.setDescription("Desc"+id); dsv.getFileMetadatas().add(fm); return fm; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index bf292723ba5..61977f6446c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5190,20 +5190,28 @@ public void testCompareDatasetVersionsAPI() throws InterruptedException { String authority = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.authority"); String identifier = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.identifier"); String datasetPersistentId = protocol + ":" + authority + "/" + identifier; - + // used for all added files + JsonObjectBuilder json = Json.createObjectBuilder() + .add("description", "my description") + .add("directoryLabel", "/data/subdir1/") + .add("categories", Json.createArrayBuilder() + .add("Data") + ); + JsonObject jsonObj = json.build(); String pathToFile = "src/main/webapp/resources/images/dataverse-icon-1200.png"; - Response uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + Response uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, jsonObj, apiToken); + uploadResponse.prettyPrint(); uploadResponse.then().assertThat() .statusCode(OK.getStatusCode()); Integer modifyFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); pathToFile = "src/main/webapp/resources/images/dataverseproject_logo.jpg"; - uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, jsonObj, apiToken); uploadResponse.then().assertThat() .statusCode(OK.getStatusCode()); Integer deleteFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); pathToFile = "src/main/webapp/resources/images/fav/favicon-16x16.png"; - uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, apiToken); + uploadResponse = UtilIT.uploadFileViaNative(String.valueOf(datasetId), pathToFile, jsonObj, apiToken); uploadResponse.then().assertThat() .statusCode(OK.getStatusCode()); Integer replaceFileId = UtilIT.getDataFileIdFromResponse(uploadResponse); @@ -5221,7 +5229,7 @@ public void testCompareDatasetVersionsAPI() throws InterruptedException { // Test adding a file pathToFile = "src/test/resources/tab/test.tab"; - Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile, Json.createObjectBuilder().build(), apiToken); + Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile, jsonObj, apiToken); uploadTabularFileResponse.prettyPrint(); uploadTabularFileResponse.then().assertThat() .statusCode(OK.getStatusCode()); @@ -5243,7 +5251,7 @@ public void testCompareDatasetVersionsAPI() throws InterruptedException { .statusCode(NO_CONTENT.getStatusCode()); // Test Replacing a file - Response replaceResponse = UtilIT.replaceFile(String.valueOf(replaceFileId), "src/main/webapp/resources/images/fav/favicon-32x32.png", apiToken); + Response replaceResponse = UtilIT.replaceFile(String.valueOf(replaceFileId), "src/main/webapp/resources/images/fav/favicon-32x32.png", jsonObj, apiToken); replaceResponse.prettyPrint(); replaceResponse.then().assertThat() .statusCode(OK.getStatusCode()); @@ -5273,6 +5281,8 @@ public void testCompareDatasetVersionsAPI() throws InterruptedException { .body("data.metadataChanges[1].changed[0].oldValue", CoreMatchers.containsString("")) .body("data.metadataChanges[1].changed[0].newValue", CoreMatchers.containsString("Parallel Group Design; Nested Case Control Design")) .body("data.filesAdded[0].fileName", CoreMatchers.equalTo("test.tab")) + .body("data.filesAdded[0].filePath", CoreMatchers.equalTo("data/subdir1")) + .body("data.filesAdded[0].description", CoreMatchers.equalTo("my description")) .body("data.filesAdded[0].tags[0]", CoreMatchers.equalTo("Survey")) .body("data.filesRemoved[0].fileName", CoreMatchers.equalTo("dataverseproject_logo.jpg")) .body("data.fileChanges[0].fileName", CoreMatchers.equalTo("dataverse-icon-1200.png")) From fa0389d87992562822518878f2cb01065f483f76 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:40:57 -0500 Subject: [PATCH 6/7] adding check for dataset order incorrect --- doc/sphinx-guides/source/api/native-api.rst | 3 +++ src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 +++ src/main/java/propertyFiles/Bundle.properties | 1 + src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 6 ++++++ 4 files changed, 13 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5e4f9debe6b..371b1440d8d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1698,6 +1698,9 @@ Compare Versions of a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Returns a list of fields that have changed between 2 Dataset versions within the Metadata and Terms of Access. Also includes the files that have been added or removed as well as files that have been modified. +When compare includes an unpublished/draft version the api token must be associated with a user having view unpublished privileges +An error will be returned if VERSION0 was not created before VERSION1 + .. code-block:: bash export SERVER_URL=https://demo.dataverse.org diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d7b0c78e611..d60f797f35c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3003,6 +3003,9 @@ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathPa DataverseRequest req = createDataverseRequest(getRequestUser(crc)); DatasetVersion dsv1 = getDatasetVersionOrDie(req, versionId1, findDatasetOrDie(id), uriInfo, headers); DatasetVersion dsv2 = getDatasetVersionOrDie(req, versionId2, findDatasetOrDie(id), uriInfo, headers); + if (dsv1.getCreateTime().getTime() > dsv2.getCreateTime().getTime()) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("dataset.version.compare.incorrect.order")); + } return ok(DatasetVersion.compareVersions(dsv1, dsv2)); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 149e6a7e828..461de5c49de 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2498,6 +2498,7 @@ dataset.version.file.changed=Files (Changed File Metadata: {0} dataset.version.file.changed2=; Changed File Metadata: {0} dataset.version.variablemetadata.changed=Variable Metadata (Changed Variable Metadata: {0} dataset.version.variablemetadata.changed2=; Changed Variable Metadata: {0} +dataset.version.compare.incorrect.order=Compare requires the older dataset version to be listed first. #DataversePage.java dataverse.item.required=Required diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 61977f6446c..9397b2246ec 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5291,5 +5291,11 @@ public void testCompareDatasetVersionsAPI() throws InterruptedException { .body("data.filesReplaced[0].newFile.fileName", CoreMatchers.equalTo("favicon-32x32.png")) .body("data.TermsOfAccess", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); + + compareResponse = UtilIT.compareDatasetVersions(datasetPersistentId, ":draft", ":latest-published", apiToken); + compareResponse.prettyPrint(); + compareResponse.then().assertThat() + .body("message", CoreMatchers.equalTo(BundleUtil.getStringFromBundle("dataset.version.compare.incorrect.order"))) + .statusCode(BAD_REQUEST.getStatusCode()); } } From 91fef44c0080a5c505a6e170489762d4982e4f43 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:44:43 -0500 Subject: [PATCH 7/7] adding check for dataset order incorrect --- .../10888-add-api-for-comparing-dataset-versions.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md b/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md index f9b3822d29d..b82441ee11a 100644 --- a/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md +++ b/doc/release-notes/10888-add-api-for-comparing-dataset-versions.md @@ -3,6 +3,8 @@ The following API have been added: /api/datasets/{persistentId}/versions/{versionId0}/compare/{versionId1} This API lists the changes between 2 dataset versions. The Json response shows the changes per field within the Metadata block and the Terms Of Access. Also listed are the files that have been added or removed. Files that have been modified will also display the new file data plus the fields that have been modified. +When compare includes an unpublished/draft version the api token must be associated with a user having view unpublished privileges +An error will be returned if VERSION0 was not created before VERSION1 Example of Metadata Block field change: ```json