diff --git a/dinky-admin/src/main/java/org/dinky/interceptor/TenantInterceptor.java b/dinky-admin/src/main/java/org/dinky/interceptor/TenantInterceptor.java index 9c9513757e..f0f33912a2 100644 --- a/dinky-admin/src/main/java/org/dinky/interceptor/TenantInterceptor.java +++ b/dinky-admin/src/main/java/org/dinky/interceptor/TenantInterceptor.java @@ -27,6 +27,7 @@ import org.apache.commons.collections4.CollectionUtils; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -68,9 +69,10 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons } int finalTenantId = Integer.parseInt(cookie.getValue()); - List tenants = userInfo.getTenantList().stream() - .filter(t -> t.getId() == finalTenantId) - .collect(Collectors.toList()); + List tenants = + Opt.ofNullable(userInfo.getTenantList()).orElse(new ArrayList<>()).stream() + .filter(t -> t.getId() == finalTenantId) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(tenants)) { StpUtil.logout(StpUtil.getLoginIdAsInt()); return false; diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/TokenService.java b/dinky-admin/src/main/java/org/dinky/service/impl/TokenService.java index 381f8c14db..db37af9a49 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/TokenService.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/TokenService.java @@ -23,8 +23,10 @@ import org.dinky.data.dto.UserDTO; import org.dinky.data.model.SysToken; import org.dinky.data.model.User; +import org.dinky.mapper.TenantMapper; import org.dinky.mapper.TokenMapper; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -53,6 +55,7 @@ public class TokenService implements SaTokenDao { private final TokenMapper tokenMapper; private final StpLogic stpLogic; + private final TenantMapper tenantMapper; /** * 存储数据的集合 @@ -307,6 +310,7 @@ public void init() { User user = new User(); user.setId(userId); userInfo.setUser(user); + userInfo.setTenantList(Collections.singletonList(tenantMapper.selectById(sysToken.getTenantId()))); UserInfoContextHolder.set(userId, userInfo); if (sysToken.getExpireType() == 1) { expireMap.put(stpLogic.splicingKeyTokenValue(sysToken.getTokenValue()), NEVER_EXPIRE); diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/BaseResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/BaseResourceManager.java index 6f58ee7757..a73c4fd888 100644 --- a/dinky-admin/src/main/java/org/dinky/service/resource/BaseResourceManager.java +++ b/dinky-admin/src/main/java/org/dinky/service/resource/BaseResourceManager.java @@ -23,6 +23,7 @@ import org.dinky.service.resource.impl.HdfsResourceManager; import org.dinky.service.resource.impl.OssResourceManager; +import java.io.File; import java.io.InputStream; import org.springframework.web.multipart.MultipartFile; @@ -39,6 +40,8 @@ public interface BaseResourceManager { void putFile(String path, MultipartFile file); + void putFile(String path, File file); + String getFileContent(String path); InputStream readFile(String path); diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/ResourcesService.java b/dinky-admin/src/main/java/org/dinky/service/resource/ResourcesService.java index 5d0768c8b8..691384aa87 100644 --- a/dinky-admin/src/main/java/org/dinky/service/resource/ResourcesService.java +++ b/dinky-admin/src/main/java/org/dinky/service/resource/ResourcesService.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.function.Function; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import com.baomidou.mybatisplus.extension.service.IService; @@ -43,6 +44,8 @@ public interface ResourcesService extends IService { */ TreeNodeDTO createFolder(Integer pid, String fileName, String desc); + TreeNodeDTO createFolderOrGet(Integer pid, String fileName, String desc); + /** * Rename an existing folder with the specified parameters. * @@ -86,6 +89,9 @@ public interface ResourcesService extends IService { */ File getFile(Integer id); + @Transactional(rollbackFor = Exception.class) + void uploadFile(Integer pid, String desc, File file); + /** * Upload a file to the specified folder. * diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/HdfsResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/HdfsResourceManager.java index 5fb0504008..264a5d047e 100644 --- a/dinky-admin/src/main/java/org/dinky/service/resource/impl/HdfsResourceManager.java +++ b/dinky-admin/src/main/java/org/dinky/service/resource/impl/HdfsResourceManager.java @@ -26,11 +26,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import java.io.File; import java.io.IOException; import java.io.InputStream; import org.springframework.web.multipart.MultipartFile; +import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; public class HdfsResourceManager implements BaseResourceManager { @@ -66,6 +68,18 @@ public void putFile(String path, MultipartFile file) { } } + @Override + public void putFile(String path, File file) { + try { + FSDataOutputStream stream = getHdfs().create(new Path(getFilePath(path)), true); + stream.write(FileUtil.readBytes(file)); + stream.flush(); + stream.close(); + } catch (IOException e) { + throw BusException.valueOf("file.upload.failed", e); + } + } + @Override public String getFileContent(String path) { return IoUtil.readUtf8(readFile(path)); diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java index b15976396e..ca964956b7 100644 --- a/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java +++ b/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java @@ -24,6 +24,7 @@ import org.dinky.service.resource.BaseResourceManager; import org.dinky.utils.OssTemplate; +import java.io.File; import java.io.InputStream; import org.springframework.web.multipart.MultipartFile; @@ -31,6 +32,7 @@ import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectRequest; +import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; public class OssResourceManager implements BaseResourceManager { @@ -63,6 +65,16 @@ public void putFile(String path, MultipartFile file) { } } + @Override + public void putFile(String path, File file) { + try { + getOssTemplate() + .putObject(getOssTemplate().getBucketName(), getFilePath(path), FileUtil.getInputStream(file)); + } catch (Exception e) { + throw new DinkyException(e); + } + } + @Override public String getFileContent(String path) { return IoUtil.readUtf8(readFile(path)); diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java index 6edab021fa..17f0bde9f6 100644 --- a/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -75,6 +76,30 @@ public TreeNodeDTO createFolder(Integer pid, String fileName, String desc) { return convertTree(resources); } + @Override + public TreeNodeDTO createFolderOrGet(Integer pid, String fileName, String desc) { + String path = "/" + fileName; + Resources resources; + long count = count( + new LambdaQueryWrapper().eq(Resources::getPid, pid).eq(Resources::getFileName, fileName)); + if (count > 0) { + resources = getOne(new LambdaQueryWrapper() + .eq(Resources::getPid, pid) + .eq(Resources::getFileName, fileName)); + } else { + resources = new Resources(); + resources.setPid(pid); + resources.setFileName(fileName); + resources.setIsDirectory(true); + resources.setType(0); + resources.setFullName(pid < 1 ? path : getById(pid).getFullName() + path); + resources.setSize(0L); + resources.setDescription(desc); + save(resources); + } + return convertTree(resources); + } + @Override @Transactional(rollbackFor = Exception.class) public void rename(Integer id, String fileName, String desc) { @@ -174,15 +199,29 @@ public File getFile(Integer id) { @Transactional(rollbackFor = Exception.class) @Override - public void uploadFile(Integer pid, String desc, MultipartFile file) { + public void uploadFile(Integer pid, String desc, File file) { Resources pResource = RESOURCES_CACHE.get(pid, () -> getById(pid)); if (!pResource.getIsDirectory()) { RESOURCES_CACHE.remove(pid); Integer realPid = pResource.getPid(); pResource = RESOURCES_CACHE.get(pid, () -> getById(realPid)); } - long size = file.getSize(); - String fileName = file.getOriginalFilename(); + long size = file.length(); + String fileName = file.getName(); + upload(pid, desc, (fullName) -> getBaseResourceManager().putFile(fullName, file), fileName, pResource, size); + } + + /** + * + * @param pid pid + * @param desc desc + * @param uploadAction uploadAction + * @param fileName fileName + * @param pResource pResource + * @param size size + */ + private void upload( + Integer pid, String desc, Consumer uploadAction, String fileName, Resources pResource, long size) { Resources currentUploadResource = getOne( new LambdaQueryWrapper().eq(Resources::getPid, pid).eq(Resources::getFileName, fileName)); String fullName; @@ -205,13 +244,27 @@ public void uploadFile(Integer pid, String desc, MultipartFile file) { resources.setDescription(desc); saveOrUpdate(resources); } - getBaseResourceManager().putFile(fullName, file); + uploadAction.accept(fullName); List resourceByPidToParent = getResourceByPidToParent(new ArrayList<>(), pid); resourceByPidToParent.forEach(x -> x.setSize(x.getSize() + size)); updateBatchById(resourceByPidToParent); } + @Transactional(rollbackFor = Exception.class) + @Override + public void uploadFile(Integer pid, String desc, MultipartFile file) { + Resources pResource = RESOURCES_CACHE.get(pid, () -> getById(pid)); + if (!pResource.getIsDirectory()) { + RESOURCES_CACHE.remove(pid); + Integer realPid = pResource.getPid(); + pResource = RESOURCES_CACHE.get(pid, () -> getById(realPid)); + } + long size = file.getSize(); + String fileName = file.getOriginalFilename(); + upload(pid, desc, (fullName) -> getBaseResourceManager().putFile(fullName, file), fileName, pResource, size); + } + @Transactional(rollbackFor = Exception.class) @Override public boolean remove(Integer id) { @@ -306,6 +359,7 @@ public List getResourcesTree() { * @param filterFunction filter function * @return {@link Result}< {@link List}< {@link Resources}>>} */ + @Override public List getResourcesTreeByFilter(Function filterFunction) { List list = this.list(); return buildResourcesTree( @@ -388,7 +442,7 @@ private List getChildList(List list, Resources resources) * @return */ private boolean hasChild(List resourcesList, Resources resources) { - return getChildList(resourcesList, resources).size() > 0; + return !getChildList(resourcesList, resources).isEmpty(); } private BaseResourceManager getBaseResourceManager() { diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfClassStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfClassStepSse.java index 7887e8b4de..83802aca04 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfClassStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfClassStepSse.java @@ -24,8 +24,8 @@ import org.dinky.data.model.GitProject; import org.dinky.function.util.UDFUtil; import org.dinky.sse.StepSse; +import org.dinky.utils.URLUtils; -import java.io.File; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Comparator; @@ -70,7 +70,7 @@ public void exec() { throw new DinkyException("flink dependency not found"); } pathList.parallelStream().forEach(jar -> { - List> udfClassByJar = UDFUtil.getUdfClassByJar(new File(jar)); + List> udfClassByJar = UDFUtil.getUdfClassByJar(URLUtils.toFile(jar)); udfMap.put(jar, udfClassByJar); sendMsg(Dict.create().set(jar, udfClassByJar)); }); diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfPythonStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfPythonStepSse.java index c5d62f1f86..18abff2a4b 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfPythonStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfPythonStepSse.java @@ -53,16 +53,16 @@ public AnalysisUdfPythonStepSse( @Override public void exec() { File zipFile = (File) params.get("zipFile"); - File projectFile = (File) params.get("projectFile"); + String zipFilePath = params.getStr("zipFilePath"); try { Thread.currentThread().getContextClassLoader().loadClass("org.apache.flink.table.api.ValidationException"); } catch (ClassNotFoundException e) { throw new DinkyException("flink dependency not found"); } - List pythonUdfList = UDFUtil.getPythonUdfList( - SystemConfiguration.getInstances().getPythonHome(), projectFile.getAbsolutePath()); + List pythonUdfList = + UDFUtil.getPythonUdfList(SystemConfiguration.getInstances().getPythonHome(), zipFile.getAbsolutePath()); GitAnalysisJarDTO gitAnalysisJarDTO = new GitAnalysisJarDTO(); - gitAnalysisJarDTO.setJarPath(zipFile.getAbsolutePath()); + gitAnalysisJarDTO.setJarPath(zipFilePath); gitAnalysisJarDTO.setClassList(pythonUdfList); List dataList = CollUtil.newArrayList(gitAnalysisJarDTO); diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java index 4755cd4b27..9dc4993977 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java @@ -19,6 +19,9 @@ package org.dinky.sse.git; +import org.dinky.data.dto.TreeNodeDTO; +import org.dinky.data.model.GitProject; +import org.dinky.service.resource.ResourcesService; import org.dinky.sse.StepSse; import org.dinky.utils.MavenUtil; @@ -30,7 +33,9 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; +import cn.hutool.core.convert.Convert; import cn.hutool.core.lang.Dict; +import cn.hutool.extra.spring.SpringUtil; /** * @author ZackYoung @@ -51,10 +56,22 @@ public GetJarsStepSse( @Override public void exec() { List jars = MavenUtil.getJars((File) params.get("pom")); - - List pathList = jars.stream().map(File::getAbsolutePath).collect(Collectors.toList()); + List pathList = uploadResources(jars); addFileMsg(pathList); params.put("jarPath", pathList); } + + private List uploadResources(List jars) { + GitProject gitProject = (GitProject) params.get("gitProject"); + + ResourcesService resourcesService = SpringUtil.getBean(ResourcesService.class); + TreeNodeDTO gitFolder = resourcesService.createFolderOrGet(0, "git", ""); + TreeNodeDTO treeNodeDTO = + resourcesService.createFolderOrGet(Convert.toInt(gitFolder.getId()), gitProject.getName(), ""); + return jars.stream() + .peek(f -> resourcesService.uploadFile(Convert.toInt(treeNodeDTO.getId()), "", f)) + .map(x -> "rs:/git/" + gitProject.getName() + "/" + x.getName()) + .collect(Collectors.toList()); + } } diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/PythonZipStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/PythonZipStepSse.java index 146d2d5d42..28b5ef0990 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/PythonZipStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/PythonZipStepSse.java @@ -19,7 +19,9 @@ package org.dinky.sse.git; +import org.dinky.data.dto.TreeNodeDTO; import org.dinky.data.model.GitProject; +import org.dinky.service.resource.ResourcesService; import org.dinky.sse.StepSse; import org.dinky.utils.GitRepository; @@ -30,9 +32,11 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; +import cn.hutool.core.convert.Convert; import cn.hutool.core.io.FileUtil; import cn.hutool.core.lang.Dict; import cn.hutool.core.util.ZipUtil; +import cn.hutool.extra.spring.SpringUtil; public class PythonZipStepSse extends StepSse { public PythonZipStepSse( @@ -51,8 +55,19 @@ public void exec() { File file = FileUtil.file(GitRepository.getProjectDir(gitProject.getName()), gitProject.getBranch()); File zipFile = ZipUtil.zip(file); + String zipFilePath = uploadResources(zipFile, gitProject); addFileMsgLog("ZipFile Path is: " + zipFile); params.put("zipFile", zipFile); + params.put("zipFilePath", zipFilePath); params.put("projectFile", file); } + + private String uploadResources(File zipFile, GitProject gitProject) { + ResourcesService resourcesService = SpringUtil.getBean(ResourcesService.class); + TreeNodeDTO gitFolder = resourcesService.createFolderOrGet(0, "git", ""); + TreeNodeDTO treeNodeDTO = + resourcesService.createFolderOrGet(Convert.toInt(gitFolder.getId()), gitProject.getName(), ""); + resourcesService.uploadFile(Convert.toInt(treeNodeDTO.getId()), "", zipFile); + return "rs:/git/" + gitProject.getName() + "/" + zipFile.getName(); + } } diff --git a/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java b/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java index eb41b3741c..9dd5023c6e 100644 --- a/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java +++ b/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java @@ -161,7 +161,7 @@ public static String getMavenCommandLineByMvn( settingsPath = StrUtil.wrap(settingsPath, "\""); List commandLine = new LinkedList<>(); - commandLine.add(mavenHome + "/bin/mvn"); + commandLine.add((mavenHome + "/bin/mvn").replaceAll(" ", "\" \"")); commandLine.add("-Dmaven.multiModuleProjectDirectory=" + projectDir); commandLine.add("-Dmaven.home=" + StrUtil.wrap(mavenHome, "\"")); Opt.ofBlankAble(repositoryDir) diff --git a/dinky-admin/src/main/java/org/dinky/utils/RuntimeUtils.java b/dinky-admin/src/main/java/org/dinky/utils/RuntimeUtils.java index 87f4ebeeaf..0a40154f6e 100644 --- a/dinky-admin/src/main/java/org/dinky/utils/RuntimeUtils.java +++ b/dinky-admin/src/main/java/org/dinky/utils/RuntimeUtils.java @@ -31,6 +31,7 @@ import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.lang.Opt; +import cn.hutool.system.SystemUtil; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -41,11 +42,13 @@ public static void run(String shell) { run(shell, log::info, log::error); } + public static final String SHELL = SystemUtil.getOsInfo().isWindows() ? "cmd /c " : "/bin/bash -c "; + public static int run(String shell, Consumer outputConsumer, Consumer errorConsumer) { Process process; int waitValue = 1; try { - process = Runtime.getRuntime().exec(shell); + process = Runtime.getRuntime().exec(SHELL + shell); RUNNING.add(process); new Thread(() -> { InputStream inputStream = process.getInputStream();