diff --git a/src/middle-server/pom.xml b/src/middle-server/pom.xml
index ec593ef..8ac66ff 100644
--- a/src/middle-server/pom.xml
+++ b/src/middle-server/pom.xml
@@ -51,8 +51,13 @@
aliyun-sdk-oss
3.15.0
-
-
+
+
+ org.apache.commons
+ commons-compress
+ 1.20
+
+
org.junit.jupiter
junit-jupiter
diff --git a/src/middle-server/src/main/java/com/serverless/middle/config/ServerConfig.java b/src/middle-server/src/main/java/com/serverless/middle/config/ServerConfig.java
index 79aad9f..49c52bb 100644
--- a/src/middle-server/src/main/java/com/serverless/middle/config/ServerConfig.java
+++ b/src/middle-server/src/main/java/com/serverless/middle/config/ServerConfig.java
@@ -32,9 +32,9 @@ public class ServerConfig extends BaseConfig {
private static final long DEFAULT_PROJECTOR_CONNECTION_TIMEOUT = 1000 * 20;
- public static final String IDEA_PROJECT_PATCH = "/root/IdeaProjects/";
+ public static final String IDEA_PROJECT_PATCH = "/root/IdeaProjects";
- public static final String IDEA_CONFIG_PATCH = "/root/.config/JetBrains/";
+ public static final String IDEA_CONFIG_PATCH = "/root/.config";
/**
* get netty boss thread size
diff --git a/src/middle-server/src/main/java/com/serverless/middle/manager/ConnectionManager.java b/src/middle-server/src/main/java/com/serverless/middle/manager/ConnectionManager.java
index 5b75c5b..b038742 100644
--- a/src/middle-server/src/main/java/com/serverless/middle/manager/ConnectionManager.java
+++ b/src/middle-server/src/main/java/com/serverless/middle/manager/ConnectionManager.java
@@ -99,8 +99,7 @@ public void setUnConnectioned(Channel channel) {
public void initIdea(String workspace, String accessKeyId, String secretKey, String securityToken, String endPoint) {
OSSManager ossManager = WORKSPACE_OSS_MAP.computeIfAbsent(workspace,
(k) -> new OSSManager(endPoint, accessKeyId, secretKey, workspace, securityToken));
- ossManager.batchDownload(IDEA_PROJECT_PATCH, IDEA_PROJECT_PATCH);
- ossManager.batchDownload(IDEA_CONFIG_PATCH, IDEA_CONFIG_PATCH);
+ ossManager.batchDownload();
}
public void shutdown() {
@@ -108,8 +107,8 @@ public void shutdown() {
uploadIdea.shutdown();
WORKSPACE_OSS_MAP.entrySet().stream().forEach(entry -> {
OSSManager ossManager = entry.getValue();
- ossManager.batchUploadFile(new File(IDEA_PROJECT_PATCH));
- ossManager.batchUploadFile(new File(IDEA_CONFIG_PATCH));
+ ossManager.batchUploadFile(IDEA_PROJECT_PATCH);
+ ossManager.batchUploadFile(IDEA_CONFIG_PATCH);
});
}
@@ -179,13 +178,13 @@ public void run() {
if (!ideaProjectorFile.exists()) {
LOGGER.info("{} doesn't exist", IDEA_PROJECT_PATCH);
} else {
- ossManager.batchUploadFile(ideaProjectorFile);
+ ossManager.batchUploadFile(IDEA_PROJECT_PATCH);
}
File ideaConfigFile = new File(IDEA_CONFIG_PATCH);
if (!ideaProjectorFile.exists()) {
LOGGER.info("{} doesn't exist", IDEA_CONFIG_PATCH);
} else {
- ossManager.batchUploadFile(ideaConfigFile);
+ ossManager.batchUploadFile(IDEA_CONFIG_PATCH);
}
});
}
diff --git a/src/middle-server/src/main/java/com/serverless/middle/oss/OSSManager.java b/src/middle-server/src/main/java/com/serverless/middle/oss/OSSManager.java
index f257f5a..9dc3df6 100644
--- a/src/middle-server/src/main/java/com/serverless/middle/oss/OSSManager.java
+++ b/src/middle-server/src/main/java/com/serverless/middle/oss/OSSManager.java
@@ -6,10 +6,12 @@
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectListing;
import com.aliyun.oss.model.PutObjectRequest;
+import com.serverless.middle.utils.TarUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.util.List;
public class OSSManager {
@@ -30,6 +32,10 @@ public class OSSManager {
private static OSSClientBuilder ossClientBuilder = new OSSClientBuilder();
+ private static final String TEMP_PATH = "/root/IdeaTemp/";
+
+ private static final String IDEA_PATH = "/root/";
+
public OSSManager(String endpoint, String accessKeyId, String accessKeySecret, String workspace, String securityToken) {
this.endpoint = endpoint;
this.accessKeyId = accessKeyId;
@@ -42,23 +48,26 @@ public OSSManager(String endpoint, String accessKeyId, String accessKeySecret, S
this(endpoint, accessKeyId, accessKeySecret, workspace, null);
}
- public void batchUploadFile(File parentFile) {
- LOGGER.info("will upload path：{}", parentFile.getAbsolutePath());
+ public void batchUploadFile(String parent) {
+ LOGGER.info("will upload path：{}", parent);
+ File parentFile = new File(parent);
+ String tarName = parentFile.getName() + ".tar.gz";
+ String tarPath = TEMP_PATH + tarName;
OSS ossClient = ossClientBuilder.build(endpoint, accessKeyId, accessKeySecret, securityToken);
try {
- if (parentFile.isDirectory()) {
- File[] files = parentFile.listFiles();
- for (File file : files) {
- LOGGER.info("will upload {} to {}", file.getAbsolutePath(), workspace + "/" + file.getAbsolutePath());
- ossClient.putObject(new PutObjectRequest(bucketName, workspace + "/" + file.getAbsolutePath(), file));
- }
- } else {
- ossClient.putObject(new PutObjectRequest(bucketName, workspace + "/" + parentFile.getAbsolutePath(), parentFile));
- }
+ TarUtils.compressTar(parent, tarPath);
+ LOGGER.info("will upload {} to {}", tarPath, workspace + "/" + tarName);
+ ossClient.putObject(new PutObjectRequest(bucketName, workspace + "/" + tarName,
+ new File(tarPath)));
+
+ } catch (IOException e) {
+ e.printStackTrace();
} finally {
if (ossClient != null) {
ossClient.shutdown();
}
+ File tarFile = new File(tarPath);
+ tarFile.delete();
}
}
@@ -67,33 +76,31 @@ public void uploadFile(String path, File file) {
ossClient.putObject(new PutObjectRequest(bucketName, path, file));
}
- public void batchDownload(String prefix, String targetPath) {
- File target = new File(targetPath);
- if (target.exists()) {
- return;
- }
+ public void batchDownload() {
OSS ossClient = ossClientBuilder.build(endpoint, accessKeyId, accessKeySecret, securityToken);
LOGGER.info("endpoint:{}.accessKeyId:{}.accessKeySecret:{},securityToken:{},bucket:{}",
- endpoint,accessKeyId,accessKeySecret,securityToken,bucketName);
+ endpoint, accessKeyId, accessKeySecret, securityToken, bucketName);
try {
- String userPath = workspace + "/" + prefix;
- ObjectListing objectListing = ossClient.listObjects(bucketName, userPath);
+ ObjectListing objectListing = ossClient.listObjects(bucketName, workspace);
List objectSummaries = objectListing.getObjectSummaries();
for (OSSObjectSummary objectSummary : objectSummaries) {
LOGGER.info("will download:{}", objectSummary.getKey());
String key = objectSummary.getKey();
- String simpleFileName = getSimpleFileName(userPath, key);
- File file = new File(targetPath);
- if (!file.exists()) {
- file.mkdirs();
- }
- ossClient.getObject(new GetObjectRequest(bucketName, key),
- new File(file.getAbsolutePath() + "/" + simpleFileName));
+ String simpleFileName = getSimpleFileName(workspace, key);
+ File tempTarFile = new File(TEMP_PATH + simpleFileName);
+ ossClient.getObject(new GetObjectRequest(bucketName, key), tempTarFile);
+ TarUtils.unCompressTar(tempTarFile.getAbsolutePath(), IDEA_PATH);
}
+ } catch (IOException e) {
+ e.printStackTrace();
} finally {
if (ossClient != null) {
ossClient.shutdown();
}
+ File temp = new File(TEMP_PATH);
+ for (File file : temp.listFiles()) {
+ file.delete();
+ }
}
}
diff --git a/src/middle-server/src/main/java/com/serverless/middle/utils/TarUtils.java b/src/middle-server/src/main/java/com/serverless/middle/utils/TarUtils.java
new file mode 100644
index 0000000..0ec4813
--- /dev/null
+++ b/src/middle-server/src/main/java/com/serverless/middle/utils/TarUtils.java
@@ -0,0 +1,93 @@
+package com.serverless.middle.utils;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.zip.GZIPOutputStream;
+
+public class TarUtils {
+
+ /**
+ * compress file to .tar.gz
+ *
+ * @param sourceFolder
+ * @param tarGzPath
+ * @throws IOException
+ */
+ public static void compressTar(String sourceFolder, String tarGzPath) throws IOException {
+ createTarFile(sourceFolder, tarGzPath);
+ }
+
+ private static void createTarFile(String sourceFolder, String tarGzPath) throws IOException {
+ try (FileOutputStream fos = new FileOutputStream(tarGzPath);
+ GZIPOutputStream gos = new GZIPOutputStream(new BufferedOutputStream(fos));
+ TarArchiveOutputStream tarOs = new TarArchiveOutputStream(gos);) {
+ // 若不设置此模式，当文件名超过 100 个字节时会抛出异常，异常大致如下：
+ // is too long ( > 100 bytes)
+ tarOs.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+ addFilesToTarGZ(sourceFolder, "", tarOs);
+ } catch (IOException e) {
+ throw e;
+ }
+ }
+
+ public static void addFilesToTarGZ(String filePath, String parent, TarArchiveOutputStream tarArchive) throws IOException {
+ File file = new File(filePath);
+ // Create entry name relative to parent file path
+ String entryName = parent + file.getName();
+ // 添加 tar ArchiveEntry
+ tarArchive.putArchiveEntry(new TarArchiveEntry(file, entryName));
+ if (file.isFile()) {
+ FileInputStream fis = new FileInputStream(file);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ // 写入文件
+ IOUtils.copy(bis, tarArchive);
+ tarArchive.closeArchiveEntry();
+ bis.close();
+ } else if (file.isDirectory()) {
+ // 因为是个文件夹，无需写入内容，关闭即可
+ tarArchive.closeArchiveEntry();
+ // 读取文件夹下所有文件
+ for (File f : file.listFiles()) {
+ // 递归
+ addFilesToTarGZ(f.getAbsolutePath(), entryName + File.separator, tarArchive);
+ }
+ }
+ }
+
+ public static void unCompressTar(String sourceTarGz, String target) throws IOException {
+ File sourceTarGzFile = new File(sourceTarGz);
+ File targetDir = new File(target);
+ // decompressing *.tar.gz files to tar
+ try (GzipCompressorInputStream gis = new GzipCompressorInputStream(Files.newInputStream(sourceTarGzFile.toPath()));
+ TarArchiveInputStream tai = new TarArchiveInputStream(gis)) {
+ TarArchiveEntry entry;
+ // 将 tar 文件解压到 targetDir 目录下
+ // 将 tar.gz文件解压成tar包,然后读取tar包里的文件元组，复制文件到指定目录
+ while ((entry = tai.getNextTarEntry()) != null) {
+ if (entry.isDirectory()) {
+ continue;
+ }
+ File targetFile = new File(targetDir, entry.getName());
+ File parent = targetFile.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ // 将文件写出到解压的目录
+ IOUtils.copy(tai, Files.newOutputStream(targetFile.toPath()));
+ }
+ } catch (IOException e) {
+ throw e;
+ }
+ }
+}