From 359b0874510a4bf96aa6648c56daf18e7a7df50b Mon Sep 17 00:00:00 2001 From: Becky Reamy Date: Fri, 20 Sep 2019 13:25:36 -0400 Subject: [PATCH] Hack to handle special characters so we can release --- .gitignore | 3 + .../java/org/kpmp/RegenerateZipFiles.java | 35 +++++++---- .../org/kpmp/packages/PackageService.java | 53 +++++++++------- src/main/java/org/kpmp/zip/ZipService.java | 62 +++++++++++++++++++ .../org/kpmp/packages/PackageServiceTest.java | 5 +- 5 files changed, 124 insertions(+), 34 deletions(-) create mode 100644 src/main/java/org/kpmp/zip/ZipService.java diff --git a/.gitignore b/.gitignore index 533bc1bd..0bc85dbe 100755 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ out .idea .settings .classpath +tokens +*credentials.json + diff --git a/src/main/java/org/kpmp/RegenerateZipFiles.java b/src/main/java/org/kpmp/RegenerateZipFiles.java index d15f2420..e56c9d59 100755 --- a/src/main/java/org/kpmp/RegenerateZipFiles.java +++ b/src/main/java/org/kpmp/RegenerateZipFiles.java @@ -5,8 +5,11 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.kpmp.externalProcess.CommandBuilder; @@ -14,6 +17,7 @@ import org.kpmp.packages.CustomPackageRepository; import org.kpmp.packages.FilePathHelper; import org.kpmp.packages.PackageKeys; +import org.kpmp.zip.ZipService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -30,14 +34,16 @@ public class RegenerateZipFiles implements CommandLineRunner { private CommandBuilder commandBuilder; private ProcessExecutor processExecutor; private final Logger log = LoggerFactory.getLogger(this.getClass()); + private ZipService zipService; @Autowired public RegenerateZipFiles(CustomPackageRepository packageRepository, CommandBuilder commandBuilder, - FilePathHelper pathHelper, ProcessExecutor processExecutor) { + FilePathHelper pathHelper, ProcessExecutor processExecutor, ZipService zipService) { this.packageRepository = packageRepository; this.commandBuilder = commandBuilder; this.pathHelper = pathHelper; this.processExecutor = processExecutor; + this.zipService = zipService; } public static void main(String[] args) { @@ -59,18 +65,23 @@ public void run(String... args) throws Exception { try { File existingZipFile = new File(zipFileName); existingZipFile.delete(); - String[] zipCommand = commandBuilder.buildZipCommand(packageId, packageMetadata); - boolean success = processExecutor.executeProcess(zipCommand); - if (success) { - LocalDateTime start = LocalDateTime.ofInstant(startRezipTime.toInstant(), - ZoneId.systemDefault()); - LocalDateTime end = LocalDateTime.ofInstant(new Date().toInstant(), ZoneId.systemDefault()); - long totalTime = ChronoUnit.SECONDS.between(start, end); - log.info("Timing: " + totalTime + " seconds"); - packageRepository.updateField(packageId, PackageKeys.REGENERATE_ZIP.getKey(), false); - } else { - log.error("Unable to zip files for package " + packageId); +// String[] zipCommand = commandBuilder.buildZipCommand(packageId, packageMetadata); +// boolean success = processExecutor.executeProcess(zipCommand); + String packagePath = pathHelper.getPackagePath(packageId); + List fileNames = pathHelper.getFilenames(packagePath); + List filePaths = new ArrayList<>(); + for (String fileName : fileNames) { + filePaths.add(packagePath + File.separator + fileName); } + + Map additionalData = new HashMap(); + additionalData.put("metadata.json", packageMetadata); + zipService.createZipFile(zipFileName, filePaths, additionalData); + LocalDateTime start = LocalDateTime.ofInstant(startRezipTime.toInstant(), ZoneId.systemDefault()); + LocalDateTime end = LocalDateTime.ofInstant(new Date().toInstant(), ZoneId.systemDefault()); + long totalTime = ChronoUnit.SECONDS.between(start, end); + log.info("Timing: " + totalTime + " seconds"); + packageRepository.updateField(packageId, PackageKeys.REGENERATE_ZIP.getKey(), false); } catch (IOException e) { log.error("Unable to delete file, invalid permissions: " + zipFileName); } diff --git a/src/main/java/org/kpmp/packages/PackageService.java b/src/main/java/org/kpmp/packages/PackageService.java index 8178d6d9..fd5f00fc 100755 --- a/src/main/java/org/kpmp/packages/PackageService.java +++ b/src/main/java/org/kpmp/packages/PackageService.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,6 +25,7 @@ import org.kpmp.packages.state.State; import org.kpmp.packages.state.StateHandlerService; import org.kpmp.users.User; +import org.kpmp.zip.ZipService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -49,10 +51,12 @@ public class PackageService { private CommandBuilder commandBuilder; private ProcessExecutor processExecutor; + private ZipService zipService; + @Autowired public PackageService(PackageFileHandler packageFileHandler, FilePathHelper filePathHelper, CustomPackageRepository packageRepository, StateHandlerService stateHandler, CommandBuilder commandBuilder, - ProcessExecutor processExecutor, LoggingService logger) { + ProcessExecutor processExecutor, LoggingService logger, ZipService zipService) { this.filePathHelper = filePathHelper; this.packageFileHandler = packageFileHandler; this.packageRepository = packageRepository; @@ -60,6 +64,7 @@ public PackageService(PackageFileHandler packageFileHandler, FilePathHelper file this.commandBuilder = commandBuilder; this.processExecutor = processExecutor; this.logger = logger; + this.zipService = zipService; } public List findAllPackages() throws JSONException, IOException { @@ -127,27 +132,33 @@ public void createZipFile(String packageId, String origin, User user) throws Exc public void run() { try { String packageMetadata = packageRepository.getJSONByPackageId(packageId); - String[] zipCommand = commandBuilder.buildZipCommand(packageId, packageMetadata); - boolean success = processExecutor.executeProcess(zipCommand); - if (success) { - logger.logInfoMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".createZipFile", - zipPackage.format(new Object[] { "Zip file created for package: ", packageId })); - long zipDuration = calculateDurationInSeconds(finishUploadTime, new Date()); - logger.logInfoMessage(PackageService.class, user, packageId, - PackageService.class.getSimpleName() + ".createZipFile", - zipTiming.format(new Object[] { packageInfo.getCreatedAt(), user.toString(), packageId, - packageInfo.getAttachments().size(), displaySize, zipDuration + " seconds" })); - - stateHandler.sendStateChange(packageId, uploadSucceededState); - - stateHandler.sendNotification(packageId, packageInfo.getPackageType(), - packageInfo.getCreatedAt(), packageInfo.getSubmitter().getFirstName(), - packageInfo.getSubmitter().getLastName(), packageInfo.getSubjectId(), origin); - } else { - logger.logErrorMessage(PackageService.class, user, packageId, - PackageService.class.getSimpleName(), "Unable to zip package"); +// String[] zipCommand = commandBuilder.buildZipCommand(packageId, packageMetadata); + String zipFileName = filePathHelper.getZipFileName(packageId); + String packagePath = filePathHelper.getPackagePath(packageId); + List fileNames = filePathHelper.getFilenames(packagePath); + List filePaths = new ArrayList<>(); + for (String fileName : fileNames) { + filePaths.add(packagePath + File.separator + fileName); } + + Map additionalData = new HashMap(); + additionalData.put("metadata.json", packageMetadata); + zipService.createZipFile(zipFileName, filePaths, additionalData); + + logger.logInfoMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".createZipFile", + zipPackage.format(new Object[] { "Zip file created for package: ", packageId })); + long zipDuration = calculateDurationInSeconds(finishUploadTime, new Date()); + logger.logInfoMessage(PackageService.class, user, packageId, + PackageService.class.getSimpleName() + ".createZipFile", + zipTiming.format(new Object[] { packageInfo.getCreatedAt(), user.toString(), packageId, + packageInfo.getAttachments().size(), displaySize, zipDuration + " seconds" })); + + stateHandler.sendStateChange(packageId, uploadSucceededState); + + stateHandler.sendNotification(packageId, packageInfo.getPackageType(), packageInfo.getCreatedAt(), + packageInfo.getSubmitter().getFirstName(), packageInfo.getSubmitter().getLastName(), + packageInfo.getSubjectId(), origin); } catch (Exception e) { logger.logErrorMessage(PackageService.class, user, packageId, PackageService.class.getSimpleName(), e.getMessage()); diff --git a/src/main/java/org/kpmp/zip/ZipService.java b/src/main/java/org/kpmp/zip/ZipService.java new file mode 100644 index 00000000..06aa663d --- /dev/null +++ b/src/main/java/org/kpmp/zip/ZipService.java @@ -0,0 +1,62 @@ +package org.kpmp.zip; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.springframework.stereotype.Component; + +@Component +public class ZipService { + + private static final String TMP_FILE_EXTENSION = ".tmp"; + + public void createZipFile(String zipFilePath, List filePaths, Map additionalFileInformation) + throws IOException { + File tempZipFileHandle = new File(zipFilePath + TMP_FILE_EXTENSION); + try (ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new File(zipFilePath))) { + zipFile.setMethod(ZipArchiveOutputStream.DEFLATED); + zipFile.setEncoding(StandardCharsets.UTF_8.name()); + for (String filePath : filePaths) { + File file = new File(filePath); + ZipArchiveEntry entry = new ZipArchiveEntry(file.getName()); + entry.setSize(file.length()); + zipFile.putArchiveEntry(entry); + try (FileInputStream fileInputStream = new FileInputStream(file)) { + try (BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { + byte[] buffer = new byte[32768]; + int data = 0; + while ((data = bufferedInputStream.read(buffer, 0, buffer.length)) != -1) { + zipFile.write(buffer, 0, data); + } + zipFile.flush(); + bufferedInputStream.close(); + } + fileInputStream.close(); + } + zipFile.closeArchiveEntry(); + } + + Set keys = additionalFileInformation.keySet(); + for (String additionalFileName : keys) { + ZipArchiveEntry additionalEntry = new ZipArchiveEntry(additionalFileName); + String additionaFileContents = additionalFileInformation.get(additionalFileName); + additionalEntry.setSize(additionaFileContents.getBytes().length); + zipFile.putArchiveEntry(additionalEntry); + zipFile.write(additionaFileContents.getBytes(StandardCharsets.UTF_8)); + zipFile.closeArchiveEntry(); + } + + File zipFileHandle = new File(zipFilePath); + tempZipFileHandle.renameTo(zipFileHandle); + } + } + +} diff --git a/src/test/java/org/kpmp/packages/PackageServiceTest.java b/src/test/java/org/kpmp/packages/PackageServiceTest.java index e95476cc..47a018e7 100755 --- a/src/test/java/org/kpmp/packages/PackageServiceTest.java +++ b/src/test/java/org/kpmp/packages/PackageServiceTest.java @@ -30,6 +30,7 @@ import org.kpmp.packages.state.State; import org.kpmp.packages.state.StateHandlerService; import org.kpmp.users.User; +import org.kpmp.zip.ZipService; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.slf4j.LoggerFactory; @@ -57,12 +58,14 @@ public class PackageServiceTest { private CommandBuilder commandBuilder; @Mock private ProcessExecutor processExecutor; + @Mock + private ZipService zipWorker; @Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); service = new PackageService(packageFileHandler, filePathHelper, packageRepository, stateHandlerService, - commandBuilder, processExecutor, logger); + commandBuilder, processExecutor, logger, zipWorker); ReflectionTestUtils.setField(service, "uploadSucceededState", "UPLOAD_SUCCEEDED"); }