feat(server): handle upload zip with directories (#537)

ia-toki · Oct 21, 2023 · 251224c · 251224c
1 parent 6c5ed58
commit 251224c
Show file tree

Hide file tree

Showing 12 changed files with 36 additions and 27 deletions.
diff --git a/judgels-backends/judgels-commons/judgels-fs/build.gradle b/judgels-backends/judgels-commons/judgels-fs/build.gradle
@@ -7,6 +7,7 @@ dependencies {
     implementation "com.amazonaws:aws-java-sdk-s3:$awsJavaSdkS3Version"
     implementation "com.github.ben-manes.caffeine:caffeine:$caffeineVersion"
     implementation "com.google.guava:guava:$guavaVersion"
+    implementation "commons-io:commons-io:$apacheCommonsIoVersion"
 
     compileOnly "com.google.dagger:dagger-compiler:$daggerVersion"
     compileOnly "org.immutables:value-annotations:$immutablesVersion"

diff --git a/judgels-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/FileSystem.java b/judgels-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/FileSystem.java
@@ -15,7 +15,7 @@ public interface FileSystem {
     String getPublicFileUrl(Path filePath);
     void uploadPrivateFile(Path filePath, InputStream content);
     String getPrivateFileUrl(Path filePath);
-    void uploadZippedFiles(Path dirPath, InputStream content, boolean includeDirectory);
+    void uploadZippedFiles(Path dirPath, InputStream content);
     List<FileInfo> listDirectoriesInDirectory(Path dirPath);
     List<FileInfo> listFilesInDirectory(Path dirPath);
     void writeByteArrayToFile(Path filePath, byte[] content);

diff --git a/judgels-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/aws/AwsFileSystem.java b/judgels-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/aws/AwsFileSystem.java
@@ -132,7 +132,7 @@ public String getPrivateFileUrl(Path filePath) {
     }
 
     @Override
-    public void uploadZippedFiles(Path dirPath, InputStream content, boolean includeDirectory) {
+    public void uploadZippedFiles(Path dirPath, InputStream content) {
         throw new UnsupportedOperationException();
     }
 

diff --git a/...backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/duplex/DuplexFileSystem.java b/...backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/duplex/DuplexFileSystem.java
@@ -66,8 +66,8 @@ public String getPrivateFileUrl(Path filePath) {
     }
 
     @Override
-    public void uploadZippedFiles(Path dirPath, InputStream content, boolean includeDirectory) {
-        local.uploadZippedFiles(dirPath, content, includeDirectory);
+    public void uploadZippedFiles(Path dirPath, InputStream content) {
+        local.uploadZippedFiles(dirPath, content);
     }
 
     @Override

diff --git a/...s-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/local/LocalFileSystem.java b/...s-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/local/LocalFileSystem.java
@@ -28,6 +28,7 @@
 import judgels.fs.FileInfo;
 import judgels.fs.FileSystem;
 import judgels.fs.NaturalFilenameComparator;
+import org.apache.commons.io.FilenameUtils;
 
 public final class LocalFileSystem implements FileSystem {
     private static final Set<String> IGNORABLE_FILES = ImmutableSet.of(
@@ -106,29 +107,36 @@ public String getPrivateFileUrl(Path filePath) {
     }
 
     @Override
-    public void uploadZippedFiles(Path dirPath, InputStream content, boolean includeDirectory) {
+    public void uploadZippedFiles(Path dirPath, InputStream content) {
         try {
             File destDir = baseDir.resolve(dirPath).toFile();
             byte[] buffer = new byte[4096];
             int entries = 0;
             long total = 0;
+
             try (ZipInputStream zis = new ZipInputStream(content)) {
                 ZipEntry ze = zis.getNextEntry();
                 while (ze != null) {
-                    String filename = ze.getName();
+                    String fileFullName = ze.getName();
+                    boolean isDirectoryEntry = ze.isDirectory();
+                    boolean isFileEntryInIgnorableDirectory = IGNORABLE_FILES.stream()
+                            .anyMatch(dir -> fileFullName.contains(dir + "/"));
+
+                    if (isDirectoryEntry || isFileEntryInIgnorableDirectory) {
+                        zis.closeEntry();
+                        ze = zis.getNextEntry();
+                        continue;
+                    }
+
+                    String filename = FilenameUtils.getName(fileFullName);
                     File file = new File(destDir, filename);
-                    if (includeDirectory && ze.isDirectory()) {
-                        file.mkdirs();
-                    } else if ((includeDirectory && file.getCanonicalPath().startsWith(destDir.getCanonicalPath()))
-                            || destDir.getAbsolutePath().equals(file.getParentFile().getAbsolutePath())) {
-                        try (FileOutputStream fos = new FileOutputStream(file)) {
-                            int len;
-                            while ((len = zis.read(buffer)) > 0) {
-                                fos.write(buffer, 0, len);
-                                total += len;
-                            }
-                            entries++;
+                    try (FileOutputStream fos = new FileOutputStream(file)) {
+                        int len;
+                        while ((len = zis.read(buffer)) > 0) {
+                            fos.write(buffer, 0, len);
+                            total += len;
                         }
+                        entries++;
                     }
 
                     zis.closeEntry();
@@ -149,7 +157,7 @@ public void uploadZippedFiles(Path dirPath, InputStream content, boolean include
 
     @Override
     public List<FileInfo> listDirectoriesInDirectory(Path dirPath) {
-        File[] files  = baseDir.resolve(dirPath).toFile().listFiles();
+        File[] files = baseDir.resolve(dirPath).toFile().listFiles();
         if (files == null) {
             return ImmutableList.of();
         }
@@ -171,7 +179,7 @@ public List<FileInfo> listDirectoriesInDirectory(Path dirPath) {
 
     @Override
     public List<FileInfo> listFilesInDirectory(Path dirPath) {
-        File[] files  = baseDir.resolve(dirPath).toFile().listFiles();
+        File[] files = baseDir.resolve(dirPath).toFile().listFiles();
         if (files == null) {
             return ImmutableList.of();
         }

diff --git a/...nds/judgels-grader-engines/src/main/java/judgels/gabriel/helpers/OutputOnlyEvaluator.java b/...nds/judgels-grader-engines/src/main/java/judgels/gabriel/helpers/OutputOnlyEvaluator.java
@@ -31,7 +31,7 @@ public void prepare(Scorer scorer, File evaluationDir, File sourceFile) throws P
         this.evaluationDir = evaluationDir;
 
         try {
-            fs.uploadZippedFiles(Paths.get(""), new FileInputStream(sourceFile), false);
+            fs.uploadZippedFiles(Paths.get(""), new FileInputStream(sourceFile));
         } catch (RuntimeException | FileNotFoundException e) {
             throw new PreparationException(e);
         }

diff --git a/...r-app/src/integTest/java/judgels/jophiel/user/avatar/UserAvatarIntegrationTestModule.java b/...r-app/src/integTest/java/judgels/jophiel/user/avatar/UserAvatarIntegrationTestModule.java
@@ -59,7 +59,7 @@ public String getPrivateFileUrl(Path filePath) {
         }
 
         @Override
-        public void uploadZippedFiles(Path dirPath, InputStream content, boolean includeDirectory) {}
+        public void uploadZippedFiles(Path dirPath, InputStream content) {}
 
         @Override
         public List<FileInfo> listDirectoriesInDirectory(Path dirPath) {

diff --git a/...ls-server-app/src/main/java/judgels/sandalphon/lesson/statement/LessonStatementStore.java b/...ls-server-app/src/main/java/judgels/sandalphon/lesson/statement/LessonStatementStore.java
@@ -136,7 +136,7 @@ public void uploadStatementMediaFile(String userJid, String lessonJid, InputStre
 
     public void uploadStatementMediaFileZipped(String userJid, String lessonJid, InputStream mediaFileZipped) {
         Path mediaDirPath = getStatementMediaDirPath(userJid, lessonJid);
-        lessonFs.uploadZippedFiles(mediaDirPath, mediaFileZipped, false);
+        lessonFs.uploadZippedFiles(mediaDirPath, mediaFileZipped);
     }
 
     public List<FileInfo> getStatementMediaFiles(String userJid, String lessonJid) {

diff --git a/...er-app/src/main/java/judgels/sandalphon/problem/base/editorial/ProblemEditorialStore.java b/...er-app/src/main/java/judgels/sandalphon/problem/base/editorial/ProblemEditorialStore.java
@@ -116,7 +116,7 @@ public void uploadEditorialMediaFile(String userJid, String problemJid, InputStr
 
     public void uploadEditorialMediaFileZipped(String userJid, String problemJid, InputStream mediaFileZipped) {
         Path mediaDirPath = getEditorialMediaDirPath(userJid, problemJid);
-        problemFs.uploadZippedFiles(mediaDirPath, mediaFileZipped, false);
+        problemFs.uploadZippedFiles(mediaDirPath, mediaFileZipped);
     }
 
     public List<FileInfo> getEditorialMediaFiles(String userJid, String problemJid) {

diff --git a/...er-app/src/main/java/judgels/sandalphon/problem/base/statement/ProblemStatementStore.java b/...er-app/src/main/java/judgels/sandalphon/problem/base/statement/ProblemStatementStore.java
@@ -141,7 +141,7 @@ public void uploadStatementMediaFile(String userJid, String problemJid, InputStr
 
     public void uploadStatementMediaFileZipped(String userJid, String problemJid, InputStream mediaFileZipped) {
         Path mediaDirPath = getStatementMediaDirPath(userJid, problemJid);
-        problemFs.uploadZippedFiles(mediaDirPath, mediaFileZipped, false);
+        problemFs.uploadZippedFiles(mediaDirPath, mediaFileZipped);
     }
 
     public List<FileInfo> getStatementMediaFiles(String userJid, String problemJid) {

diff --git a/...ver-app/src/main/java/judgels/sandalphon/problem/programming/ProgrammingProblemStore.java b/...ver-app/src/main/java/judgels/sandalphon/problem/programming/ProgrammingProblemStore.java
@@ -94,7 +94,7 @@ public void uploadGradingTestDataFile(String userJid, String problemJid, InputSt
     }
 
     public void uploadGradingTestDataFileZipped(String userJid, String problemJid, InputStream testDataFileZipped) {
-        problemFs.uploadZippedFiles(getGradingTestDataDirPath(userJid, problemJid), testDataFileZipped, false);
+        problemFs.uploadZippedFiles(getGradingTestDataDirPath(userJid, problemJid), testDataFileZipped);
 
         updateGradingLastUpdateTime(userJid, problemJid);
     }
@@ -106,7 +106,7 @@ public void uploadGradingHelperFile(String userJid, String problemJid, InputStre
     }
 
     public void uploadGradingHelperFileZipped(String userJid, String problemJid, InputStream helperFileZipped) {
-        problemFs.uploadZippedFiles(getGradingHelpersDirPath(userJid, problemJid), helperFileZipped, false);
+        problemFs.uploadZippedFiles(getGradingHelpersDirPath(userJid, problemJid), helperFileZipped);
 
         updateGradingLastUpdateTime(userJid, problemJid);
     }

diff --git a/judgels-backends/judgels-server-app/src/test/java/judgels/fs/InMemoryFileSystem.java b/judgels-backends/judgels-server-app/src/test/java/judgels/fs/InMemoryFileSystem.java
@@ -64,7 +64,7 @@ public String getPrivateFileUrl(Path filePath) {
     }
 
     @Override
-    public void uploadZippedFiles(Path dirPath, InputStream content, boolean includeDirectory) {
+    public void uploadZippedFiles(Path dirPath, InputStream content) {
         throw new UnsupportedOperationException();
     }