Skip to content
This repository has been archived by the owner on Sep 14, 2022. It is now read-only.

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
imaegoo committed Aug 3, 2022
2 parents 15fbd06 + 4ada7fd commit dafbbb0
Show file tree
Hide file tree
Showing 5 changed files with 140 additions and 36 deletions.
9 changes: 7 additions & 2 deletions src/middle-server/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,13 @@
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.15.0</version>
</dependency>


<!-- compress-->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.20</version>
</dependency>
<!-- junit-->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ public class ServerConfig extends BaseConfig {

private static final long DEFAULT_PROJECTOR_CONNECTION_TIMEOUT = 1000 * 20;

public static final String IDEA_PROJECT_PATCH = "/root/IdeaProjects/";
public static final String IDEA_PROJECT_PATCH = "/root/IdeaProjects";

public static final String IDEA_CONFIG_PATCH = "/root/.config/JetBrains/";
public static final String IDEA_CONFIG_PATCH = "/root/.config";

/**
* get netty boss thread size
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,17 +99,16 @@ public void setUnConnectioned(Channel channel) {
public void initIdea(String workspace, String accessKeyId, String secretKey, String securityToken, String endPoint) {
OSSManager ossManager = WORKSPACE_OSS_MAP.computeIfAbsent(workspace,
(k) -> new OSSManager(endPoint, accessKeyId, secretKey, workspace, securityToken));
ossManager.batchDownload(IDEA_PROJECT_PATCH, IDEA_PROJECT_PATCH);
ossManager.batchDownload(IDEA_CONFIG_PATCH, IDEA_CONFIG_PATCH);
ossManager.batchDownload();
}

public void shutdown() {
connectionCheck.shutdown();
uploadIdea.shutdown();
WORKSPACE_OSS_MAP.entrySet().stream().forEach(entry -> {
OSSManager ossManager = entry.getValue();
ossManager.batchUploadFile(new File(IDEA_PROJECT_PATCH));
ossManager.batchUploadFile(new File(IDEA_CONFIG_PATCH));
ossManager.batchUploadFile(IDEA_PROJECT_PATCH);
ossManager.batchUploadFile(IDEA_CONFIG_PATCH);
});
}

Expand Down Expand Up @@ -179,13 +178,13 @@ public void run() {
if (!ideaProjectorFile.exists()) {
LOGGER.info("{} doesn't exist", IDEA_PROJECT_PATCH);
} else {
ossManager.batchUploadFile(ideaProjectorFile);
ossManager.batchUploadFile(IDEA_PROJECT_PATCH);
}
File ideaConfigFile = new File(IDEA_CONFIG_PATCH);
if (!ideaProjectorFile.exists()) {
LOGGER.info("{} doesn't exist", IDEA_CONFIG_PATCH);
} else {
ossManager.batchUploadFile(ideaConfigFile);
ossManager.batchUploadFile(IDEA_CONFIG_PATCH);
}
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectListing;
import com.aliyun.oss.model.PutObjectRequest;
import com.serverless.middle.utils.TarUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.List;

public class OSSManager {
Expand All @@ -30,6 +32,10 @@ public class OSSManager {

private static OSSClientBuilder ossClientBuilder = new OSSClientBuilder();

private static final String TEMP_PATH = "/root/IdeaTemp/";

private static final String IDEA_PATH = "/root/";

public OSSManager(String endpoint, String accessKeyId, String accessKeySecret, String workspace, String securityToken) {
this.endpoint = endpoint;
this.accessKeyId = accessKeyId;
Expand All @@ -42,23 +48,26 @@ public OSSManager(String endpoint, String accessKeyId, String accessKeySecret, S
this(endpoint, accessKeyId, accessKeySecret, workspace, null);
}

public void batchUploadFile(File parentFile) {
LOGGER.info("will upload path:{}", parentFile.getAbsolutePath());
public void batchUploadFile(String parent) {
LOGGER.info("will upload path:{}", parent);
File parentFile = new File(parent);
String tarName = parentFile.getName() + ".tar.gz";
String tarPath = TEMP_PATH + tarName;
OSS ossClient = ossClientBuilder.build(endpoint, accessKeyId, accessKeySecret, securityToken);
try {
if (parentFile.isDirectory()) {
File[] files = parentFile.listFiles();
for (File file : files) {
LOGGER.info("will upload {} to {}", file.getAbsolutePath(), workspace + "/" + file.getAbsolutePath());
ossClient.putObject(new PutObjectRequest(bucketName, workspace + "/" + file.getAbsolutePath(), file));
}
} else {
ossClient.putObject(new PutObjectRequest(bucketName, workspace + "/" + parentFile.getAbsolutePath(), parentFile));
}
TarUtils.compressTar(parent, tarPath);
LOGGER.info("will upload {} to {}", tarPath, workspace + "/" + tarName);
ossClient.putObject(new PutObjectRequest(bucketName, workspace + "/" + tarName,
new File(tarPath)));

} catch (IOException e) {
e.printStackTrace();
} finally {
if (ossClient != null) {
ossClient.shutdown();
}
File tarFile = new File(tarPath);
tarFile.delete();
}
}

Expand All @@ -67,33 +76,31 @@ public void uploadFile(String path, File file) {
ossClient.putObject(new PutObjectRequest(bucketName, path, file));
}

public void batchDownload(String prefix, String targetPath) {
File target = new File(targetPath);
if (target.exists()) {
return;
}
public void batchDownload() {
OSS ossClient = ossClientBuilder.build(endpoint, accessKeyId, accessKeySecret, securityToken);
LOGGER.info("endpoint:{}.accessKeyId:{}.accessKeySecret:{},securityToken:{},bucket:{}",
endpoint,accessKeyId,accessKeySecret,securityToken,bucketName);
endpoint, accessKeyId, accessKeySecret, securityToken, bucketName);
try {
String userPath = workspace + "/" + prefix;
ObjectListing objectListing = ossClient.listObjects(bucketName, userPath);
ObjectListing objectListing = ossClient.listObjects(bucketName, workspace);
List<OSSObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (OSSObjectSummary objectSummary : objectSummaries) {
LOGGER.info("will download:{}", objectSummary.getKey());
String key = objectSummary.getKey();
String simpleFileName = getSimpleFileName(userPath, key);
File file = new File(targetPath);
if (!file.exists()) {
file.mkdirs();
}
ossClient.getObject(new GetObjectRequest(bucketName, key),
new File(file.getAbsolutePath() + "/" + simpleFileName));
String simpleFileName = getSimpleFileName(workspace, key);
File tempTarFile = new File(TEMP_PATH + simpleFileName);
ossClient.getObject(new GetObjectRequest(bucketName, key), tempTarFile);
TarUtils.unCompressTar(tempTarFile.getAbsolutePath(), IDEA_PATH);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (ossClient != null) {
ossClient.shutdown();
}
File temp = new File(TEMP_PATH);
for (File file : temp.listFiles()) {
file.delete();
}
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package com.serverless.middle.utils;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.utils.IOUtils;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.zip.GZIPOutputStream;

public class TarUtils {

/**
* compress file to .tar.gz
*
* @param sourceFolder
* @param tarGzPath
* @throws IOException
*/
public static void compressTar(String sourceFolder, String tarGzPath) throws IOException {
createTarFile(sourceFolder, tarGzPath);
}

private static void createTarFile(String sourceFolder, String tarGzPath) throws IOException {
try (FileOutputStream fos = new FileOutputStream(tarGzPath);
GZIPOutputStream gos = new GZIPOutputStream(new BufferedOutputStream(fos));
TarArchiveOutputStream tarOs = new TarArchiveOutputStream(gos);) {
// 若不设置此模式,当文件名超过 100 个字节时会抛出异常,异常大致如下:
// is too long ( > 100 bytes)
tarOs.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
addFilesToTarGZ(sourceFolder, "", tarOs);
} catch (IOException e) {
throw e;
}
}

public static void addFilesToTarGZ(String filePath, String parent, TarArchiveOutputStream tarArchive) throws IOException {
File file = new File(filePath);
// Create entry name relative to parent file path
String entryName = parent + file.getName();
// 添加 tar ArchiveEntry
tarArchive.putArchiveEntry(new TarArchiveEntry(file, entryName));
if (file.isFile()) {
FileInputStream fis = new FileInputStream(file);
BufferedInputStream bis = new BufferedInputStream(fis);
// 写入文件
IOUtils.copy(bis, tarArchive);
tarArchive.closeArchiveEntry();
bis.close();
} else if (file.isDirectory()) {
// 因为是个文件夹,无需写入内容,关闭即可
tarArchive.closeArchiveEntry();
// 读取文件夹下所有文件
for (File f : file.listFiles()) {
// 递归
addFilesToTarGZ(f.getAbsolutePath(), entryName + File.separator, tarArchive);
}
}
}

public static void unCompressTar(String sourceTarGz, String target) throws IOException {
File sourceTarGzFile = new File(sourceTarGz);
File targetDir = new File(target);
// decompressing *.tar.gz files to tar
try (GzipCompressorInputStream gis = new GzipCompressorInputStream(Files.newInputStream(sourceTarGzFile.toPath()));
TarArchiveInputStream tai = new TarArchiveInputStream(gis)) {
TarArchiveEntry entry;
// 将 tar 文件解压到 targetDir 目录下
// 将 tar.gz文件解压成tar包,然后读取tar包里的文件元组,复制文件到指定目录
while ((entry = tai.getNextTarEntry()) != null) {
if (entry.isDirectory()) {
continue;
}
File targetFile = new File(targetDir, entry.getName());
File parent = targetFile.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
// 将文件写出到解压的目录
IOUtils.copy(tai, Files.newOutputStream(targetFile.toPath()));
}
} catch (IOException e) {
throw e;
}
}
}

0 comments on commit dafbbb0

Please sign in to comment.