Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@jo-pol
jo-pol committed Sep 10, 2024
1 parent 699cb98 commit 2e0de94
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 34 deletions.
5 changes: 3 additions & 2 deletions src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException


if (newStorageIdentifier == null) {
if (getFilesTempDirectory() != null) {
var filesTempDirectory = getFilesTempDirectory();
if (filesTempDirectory != null) {
try {
tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
tempFile = Files.createTempFile(Paths.get(filesTempDirectory), "tmp", "upload");
// "temporary" location is the key here; this is why we are not using
// the DataStore framework for this - the assumption is that
// temp files will always be stored on the local filesystem.
Expand Down
205 changes: 187 additions & 18 deletions src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,40 @@
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
import edu.harvard.iq.dataverse.util.JhoveFileType;
import edu.harvard.iq.dataverse.util.SystemConfig;
import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;
import edu.harvard.iq.dataverse.util.testing.JvmSetting;
import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.MockedStatic;
import org.mockito.Mockito;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import static org.apache.commons.io.file.FilesUncheck.createDirectory;
import static org.apache.commons.io.file.FilesUncheck.createDirectories;
import static org.apache.commons.io.file.PathUtils.deleteDirectory;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;


@LocalJvmSettings
public class CreateNewDataFilesTest {
Expand All @@ -35,40 +49,195 @@ public void cleanTmpDir() throws IOException {
var tmpDir = Paths.get("target/test/tmp");
if(tmpDir.toFile().exists())
deleteDirectory(tmpDir);
Files.createDirectories(tmpDir);
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp")
public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoundException {
var dsVersion = Mockito.mock(DatasetVersion.class);
Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class));

// TODO fix copilot suggestion
JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class);
Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path");
mockTmpLookup();
var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockQuota(true, 0L));

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessageContaining("Failed to save the upload as a temp file (temp disk space?)")
.hasRootCauseInstanceOf(NoSuchFileException.class)
.getRootCause()
.hasMessageStartingWith("target/test/tmp/temp/tmp");
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp")
public void execute_fails_to_upload_too_big_files() throws FileNotFoundException {
createDirectories(Path.of("target/test/tmp/temp"));

mockTmpLookup();
var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockQuota(true, 1000L));

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessage("This file size (56.0 KB) exceeds the size limit of 1000 B.");
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp")
public void execute_fails_on_too_little_remaining_storage() throws FileNotFoundException {
createDirectories(Path.of("target/test/tmp/temp"));

var cmd = new CreateNewDataFilesCommand(
mockTmpLookup();
var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockQuota(true, 1000000L));
try (MockedStatic<JhoveFileType> mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessage("This file (size 56.0 KB) exceeds the remaining storage quota of 500 B.");
}
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp")
public void execute_succeeds() throws FileNotFoundException, CommandException {
var tempDir = Path.of("target/test/tmp/temp");
var testFile = "scripts/search/data/binary/trees.zip";
createDirectories(tempDir);

mockTmpLookup();
var cmd = createCmd(testFile, mockDatasetVersion());
var ctxt = mockCommandContext(mockQuota(false, 1000000L));
try (MockedStatic<JhoveFileType> mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");

var result = cmd.execute(ctxt);

assertThat(result.getErrors()).hasSize(0);
assertThat(result.getDataFiles()).hasSize(1);

var dataFile = result.getDataFiles().subList(0, 1).get(0);
var storageId = dataFile.getStorageIdentifier();

// uploaded zip remains in tmp directory
assertThat(tempDir.toFile().list()).hasSize(1);
assertThat(tempDir.resolve(storageId).toFile().length())
.isEqualTo(new File(testFile).length());
}
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/tmp")
public void execute_with_2_shapes() throws Exception {
var tempDir = Path.of("target/test/tmp/temp");
List<String> file_names = Arrays.asList("shape1.shp", "shape1.shx", "shape1.dbf", "shape1.prj", "shape1.fbn", "shape1.fbx", // 1st shapefile
"shape2.shp", "shape2.shx", "shape2.dbf", "shape2.prj", // 2nd shapefile
"shape2.txt", "shape2.pdf", "shape2", // single files, same basename as 2nd shapefile
"README.MD", "shp_dictionary.xls", "notes"); //, "prj"); // single files
File testFile = createAndZipFiles(file_names, "two-shapes.zip");

// TODO same results as for scripts/search/data/binary/trees.zip
// this one and for scripts/search/data/shape/shapefile.zip
// and neither covers ShapefileHandler.SHAPEFILE_FILE_TYPE
// despite static mock still getting a warning: Failed to run the file utility mime type check on file example.zip
createDirectories(tempDir);

mockTmpLookup();
var cmd = createCmd(testFile.toString(), mockDatasetVersion());
var ctxt = mockCommandContext(mockQuota(false, 1000000L));

try (MockedStatic<JhoveFileType> mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");
var result = cmd.execute(ctxt);

assertThat(result.getErrors()).hasSize(0);
assertThat(result.getDataFiles()).hasSize(1);

var dataFile = result.getDataFiles().subList(0, 1).get(0);
var storageId = dataFile.getStorageIdentifier();

// uploaded zip remains in tmp directory
assertThat(tempDir.toFile().list()).hasSize(1);
assertThat(tempDir.resolve(storageId).toFile().length())
.isEqualTo(testFile.length());
}
}

@TempDir
Path tempFolder;

// copied and refactored from ShapefileHandlerTest
private File createAndZipFiles(List<String> file_names, String zipfile_name) throws IOException {
if ((file_names == null) || (zipfile_name == null)) {
return null;
}

// Create blank files based on a list of file names
Collection<File> fileCollection = new ArrayList<>();
for (String fname : file_names) {
fileCollection.add(Files.createFile(tempFolder.resolve(fname)).toFile());
}

Path zip_file_obj = tempFolder.resolve(zipfile_name);
try (ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zip_file_obj.toFile()))) {

// Iterate through File objects and add them to the ZipOutputStream
for (File file_obj : fileCollection) {

FileInputStream file_input_stream = new FileInputStream(file_obj);
ZipEntry zipEntry = new ZipEntry(file_obj.getName());
zip_stream.putNextEntry(zipEntry);

byte[] bytes = new byte[1024];
int length;
while ((length = file_input_stream.read(bytes)) >= 0) {
zip_stream.write(bytes, 0, length);
}

zip_stream.closeEntry();
file_input_stream.close();
file_obj.delete(); // cleanup
}
}

return zip_file_obj.toFile();

}

private static @NotNull CreateNewDataFilesCommand createCmd(String name, DatasetVersion dsVersion) throws FileNotFoundException {
return new CreateNewDataFilesCommand(
Mockito.mock(DataverseRequest.class),
dsVersion,
new FileInputStream("scripts/search/data/shape/shapefile.zip"),
new FileInputStream(name),
"example.zip",
"application/zip",
null,
new UploadSessionQuotaLimit(1000L, 500L),
"sha");
}

private static @NotNull CommandContext mockCommandContext(SystemConfig sysCfg) {
var ctxt = Mockito.mock(CommandContext.class);
var sysCfg = Mockito.mock(SystemConfig.class);
Mockito.when(ctxt.systemConfig()).thenReturn(sysCfg);
Mockito.when(sysCfg.isStorageQuotasEnforced()).thenReturn(true);
return ctxt;
}

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessageContaining("Failed to save the upload as a temp file (temp disk space?)")
.hasRootCauseInstanceOf(NoSuchFileException.class)
.getRootCause()
.hasMessageStartingWith("target/test/tmp/temp/tmp");
private static @NotNull SystemConfig mockQuota(boolean isStorageQuataEnforced, long maxFileUploadSizeForStore) {
var sysCfg = Mockito.mock(SystemConfig.class);
Mockito.when(sysCfg.isStorageQuotasEnforced()).thenReturn(isStorageQuataEnforced);
Mockito.when(sysCfg.getMaxFileUploadSizeForStore(any())).thenReturn(maxFileUploadSizeForStore);
return sysCfg;
}

private static void mockTmpLookup() {
JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class);
Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path");
}

private static @NotNull DatasetVersion mockDatasetVersion() {
var dsVersion = Mockito.mock(DatasetVersion.class);
Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class));
return dsVersion;
}

}
15 changes: 1 addition & 14 deletions src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,22 +63,9 @@ private File createBlankFile(String filename) throws IOException {
}
return Files.createFile(tempFolder.resolve(filename)).toFile();
}

private FileInputStream createZipReturnFilestream(List<String> file_names, String zipfile_name) throws IOException{

File zip_file_obj = this.createAndZipFiles(file_names, zipfile_name);
if (zip_file_obj == null){
return null;
}

FileInputStream file_input_stream = new FileInputStream(zip_file_obj);

return file_input_stream;

}

/*
Convenience class to create .zip file and return a FileInputStream
Convenience method to create .zip file and return a File
@param List<String> file_names - List of filenames to add to .zip. These names will be used to create 0 length files
@param String zipfile_name - Name of .zip file to create
Expand Down

0 comments on commit 2e0de94

Please sign in to comment.