From 2e0de942ecc5b211a462f4e38e18056323ec4b1c Mon Sep 17 00:00:00 2001 From: jo-pol Date: Tue, 10 Sep 2024 16:08:46 +0200 Subject: [PATCH] wip --- .../impl/CreateNewDataFilesCommand.java | 5 +- .../command/impl/CreateNewDataFilesTest.java | 205 ++++++++++++++++-- .../util/shapefile/ShapefileHandlerTest.java | 15 +- 3 files changed, 191 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java index 2735c94faec..31b11a4a367 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java @@ -140,9 +140,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException if (newStorageIdentifier == null) { - if (getFilesTempDirectory() != null) { + var filesTempDirectory = getFilesTempDirectory(); + if (filesTempDirectory != null) { try { - tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); + tempFile = Files.createTempFile(Paths.get(filesTempDirectory), "tmp", "upload"); // "temporary" location is the key here; this is why we are not using // the DataStore framework for this - the assumption is that // temp files will always be stored on the local filesystem. diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java index 8da145870c1..899d81941b6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java @@ -7,26 +7,40 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit; +import edu.harvard.iq.dataverse.util.JhoveFileType; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.file.CreateDataFileResult; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; -import static org.apache.commons.io.file.FilesUncheck.createDirectory; +import static org.apache.commons.io.file.FilesUncheck.createDirectories; import static org.apache.commons.io.file.PathUtils.deleteDirectory; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; + @LocalJvmSettings public class CreateNewDataFilesTest { @@ -35,40 +49,195 @@ public void cleanTmpDir() throws IOException { var tmpDir = Paths.get("target/test/tmp"); if(tmpDir.toFile().exists()) deleteDirectory(tmpDir); - Files.createDirectories(tmpDir); } @Test @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp") public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoundException { - var dsVersion = Mockito.mock(DatasetVersion.class); - Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class)); - // TODO fix copilot suggestion - JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class); - Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path"); + mockTmpLookup(); + var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion()); + var ctxt = mockCommandContext(mockQuota(true, 0L)); + + assertThatThrownBy(() -> cmd.execute(ctxt)) + .isInstanceOf(CommandException.class) + .hasMessageContaining("Failed to save the upload as a temp file (temp disk space?)") + .hasRootCauseInstanceOf(NoSuchFileException.class) + .getRootCause() + .hasMessageStartingWith("target/test/tmp/temp/tmp"); + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp") + public void execute_fails_to_upload_too_big_files() throws FileNotFoundException { + createDirectories(Path.of("target/test/tmp/temp")); + + mockTmpLookup(); + var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion()); + var ctxt = mockCommandContext(mockQuota(true, 1000L)); + + assertThatThrownBy(() -> cmd.execute(ctxt)) + .isInstanceOf(CommandException.class) + .hasMessage("This file size (56.0 KB) exceeds the size limit of 1000 B."); + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp") + public void execute_fails_on_too_little_remaining_storage() throws FileNotFoundException { + createDirectories(Path.of("target/test/tmp/temp")); - var cmd = new CreateNewDataFilesCommand( + mockTmpLookup(); + var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion()); + var ctxt = mockCommandContext(mockQuota(true, 1000000L)); + try (MockedStatic mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { + mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + + assertThatThrownBy(() -> cmd.execute(ctxt)) + .isInstanceOf(CommandException.class) + .hasMessage("This file (size 56.0 KB) exceeds the remaining storage quota of 500 B."); + } + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp") + public void execute_succeeds() throws FileNotFoundException, CommandException { + var tempDir = Path.of("target/test/tmp/temp"); + var testFile = "scripts/search/data/binary/trees.zip"; + createDirectories(tempDir); + + mockTmpLookup(); + var cmd = createCmd(testFile, mockDatasetVersion()); + var ctxt = mockCommandContext(mockQuota(false, 1000000L)); + try (MockedStatic mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { + mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + + var result = cmd.execute(ctxt); + + assertThat(result.getErrors()).hasSize(0); + assertThat(result.getDataFiles()).hasSize(1); + + var dataFile = result.getDataFiles().subList(0, 1).get(0); + var storageId = dataFile.getStorageIdentifier(); + + // uploaded zip remains in tmp directory + assertThat(tempDir.toFile().list()).hasSize(1); + assertThat(tempDir.resolve(storageId).toFile().length()) + .isEqualTo(new File(testFile).length()); + } + } + + @Test + @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp") + public void execute_with_2_shapes() throws Exception { + var tempDir = Path.of("target/test/tmp/temp"); + List file_names = Arrays.asList("shape1.shp", "shape1.shx", "shape1.dbf", "shape1.prj", "shape1.fbn", "shape1.fbx", // 1st shapefile + "shape2.shp", "shape2.shx", "shape2.dbf", "shape2.prj", // 2nd shapefile + "shape2.txt", "shape2.pdf", "shape2", // single files, same basename as 2nd shapefile + "README.MD", "shp_dictionary.xls", "notes"); //, "prj"); // single files + File testFile = createAndZipFiles(file_names, "two-shapes.zip"); + + // TODO same results as for scripts/search/data/binary/trees.zip + // this one and for scripts/search/data/shape/shapefile.zip + // and neither covers ShapefileHandler.SHAPEFILE_FILE_TYPE + // despite static mock still getting a warning: Failed to run the file utility mime type check on file example.zip + createDirectories(tempDir); + + mockTmpLookup(); + var cmd = createCmd(testFile.toString(), mockDatasetVersion()); + var ctxt = mockCommandContext(mockQuota(false, 1000000L)); + + try (MockedStatic mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { + mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); + var result = cmd.execute(ctxt); + + assertThat(result.getErrors()).hasSize(0); + assertThat(result.getDataFiles()).hasSize(1); + + var dataFile = result.getDataFiles().subList(0, 1).get(0); + var storageId = dataFile.getStorageIdentifier(); + + // uploaded zip remains in tmp directory + assertThat(tempDir.toFile().list()).hasSize(1); + assertThat(tempDir.resolve(storageId).toFile().length()) + .isEqualTo(testFile.length()); + } + } + + @TempDir + Path tempFolder; + + // copied and refactored from ShapefileHandlerTest + private File createAndZipFiles(List file_names, String zipfile_name) throws IOException { + if ((file_names == null) || (zipfile_name == null)) { + return null; + } + + // Create blank files based on a list of file names + Collection fileCollection = new ArrayList<>(); + for (String fname : file_names) { + fileCollection.add(Files.createFile(tempFolder.resolve(fname)).toFile()); + } + + Path zip_file_obj = tempFolder.resolve(zipfile_name); + try (ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zip_file_obj.toFile()))) { + + // Iterate through File objects and add them to the ZipOutputStream + for (File file_obj : fileCollection) { + + FileInputStream file_input_stream = new FileInputStream(file_obj); + ZipEntry zipEntry = new ZipEntry(file_obj.getName()); + zip_stream.putNextEntry(zipEntry); + + byte[] bytes = new byte[1024]; + int length; + while ((length = file_input_stream.read(bytes)) >= 0) { + zip_stream.write(bytes, 0, length); + } + + zip_stream.closeEntry(); + file_input_stream.close(); + file_obj.delete(); // cleanup + } + } + + return zip_file_obj.toFile(); + + } + + private static @NotNull CreateNewDataFilesCommand createCmd(String name, DatasetVersion dsVersion) throws FileNotFoundException { + return new CreateNewDataFilesCommand( Mockito.mock(DataverseRequest.class), dsVersion, - new FileInputStream("scripts/search/data/shape/shapefile.zip"), + new FileInputStream(name), "example.zip", "application/zip", null, new UploadSessionQuotaLimit(1000L, 500L), "sha"); + } + private static @NotNull CommandContext mockCommandContext(SystemConfig sysCfg) { var ctxt = Mockito.mock(CommandContext.class); - var sysCfg = Mockito.mock(SystemConfig.class); Mockito.when(ctxt.systemConfig()).thenReturn(sysCfg); - Mockito.when(sysCfg.isStorageQuotasEnforced()).thenReturn(true); + return ctxt; + } - assertThatThrownBy(() -> cmd.execute(ctxt)) - .isInstanceOf(CommandException.class) - .hasMessageContaining("Failed to save the upload as a temp file (temp disk space?)") - .hasRootCauseInstanceOf(NoSuchFileException.class) - .getRootCause() - .hasMessageStartingWith("target/test/tmp/temp/tmp"); + private static @NotNull SystemConfig mockQuota(boolean isStorageQuataEnforced, long maxFileUploadSizeForStore) { + var sysCfg = Mockito.mock(SystemConfig.class); + Mockito.when(sysCfg.isStorageQuotasEnforced()).thenReturn(isStorageQuataEnforced); + Mockito.when(sysCfg.getMaxFileUploadSizeForStore(any())).thenReturn(maxFileUploadSizeForStore); + return sysCfg; + } + + private static void mockTmpLookup() { + JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class); + Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path"); + } + + private static @NotNull DatasetVersion mockDatasetVersion() { + var dsVersion = Mockito.mock(DatasetVersion.class); + Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class)); + return dsVersion; } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java index 587ea265f29..c5684534335 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java @@ -63,22 +63,9 @@ private File createBlankFile(String filename) throws IOException { } return Files.createFile(tempFolder.resolve(filename)).toFile(); } - - private FileInputStream createZipReturnFilestream(List file_names, String zipfile_name) throws IOException{ - - File zip_file_obj = this.createAndZipFiles(file_names, zipfile_name); - if (zip_file_obj == null){ - return null; - } - - FileInputStream file_input_stream = new FileInputStream(zip_file_obj); - return file_input_stream; - - } - /* - Convenience class to create .zip file and return a FileInputStream + Convenience method to create .zip file and return a File @param List file_names - List of filenames to add to .zip. These names will be used to create 0 length files @param String zipfile_name - Name of .zip file to create