[Feature][Admin] Git compilation package and Resource integration (#2523

) * git_add_resources * git_add_resources * git_add_resources
DataLinkDC · Nov 14, 2023 · 6f689ac · 6f689ac
1 parent e50059b
commit 6f689ac
Show file tree

Hide file tree

Showing 13 changed files with 148 additions and 18 deletions.
diff --git a/dinky-admin/src/main/java/org/dinky/interceptor/TenantInterceptor.java b/dinky-admin/src/main/java/org/dinky/interceptor/TenantInterceptor.java
@@ -27,6 +27,7 @@
 
 import org.apache.commons.collections4.CollectionUtils;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -68,9 +69,10 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons
                         }
 
                         int finalTenantId = Integer.parseInt(cookie.getValue());
-                        List<Tenant> tenants = userInfo.getTenantList().stream()
-                                .filter(t -> t.getId() == finalTenantId)
-                                .collect(Collectors.toList());
+                        List<Tenant> tenants =
+                                Opt.ofNullable(userInfo.getTenantList()).orElse(new ArrayList<>()).stream()
+                                        .filter(t -> t.getId() == finalTenantId)
+                                        .collect(Collectors.toList());
                         if (CollectionUtils.isEmpty(tenants)) {
                             StpUtil.logout(StpUtil.getLoginIdAsInt());
                             return false;

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/TokenService.java b/dinky-admin/src/main/java/org/dinky/service/impl/TokenService.java
@@ -23,8 +23,10 @@
 import org.dinky.data.dto.UserDTO;
 import org.dinky.data.model.SysToken;
 import org.dinky.data.model.User;
+import org.dinky.mapper.TenantMapper;
 import org.dinky.mapper.TokenMapper;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -53,6 +55,7 @@ public class TokenService implements SaTokenDao {
 
     private final TokenMapper tokenMapper;
     private final StpLogic stpLogic;
+    private final TenantMapper tenantMapper;
 
     /**
      * 存储数据的集合
@@ -307,6 +310,7 @@ public void init() {
             User user = new User();
             user.setId(userId);
             userInfo.setUser(user);
+            userInfo.setTenantList(Collections.singletonList(tenantMapper.selectById(sysToken.getTenantId())));
             UserInfoContextHolder.set(userId, userInfo);
             if (sysToken.getExpireType() == 1) {
                 expireMap.put(stpLogic.splicingKeyTokenValue(sysToken.getTokenValue()), NEVER_EXPIRE);

diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/BaseResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/BaseResourceManager.java
@@ -23,6 +23,7 @@
 import org.dinky.service.resource.impl.HdfsResourceManager;
 import org.dinky.service.resource.impl.OssResourceManager;
 
+import java.io.File;
 import java.io.InputStream;
 
 import org.springframework.web.multipart.MultipartFile;
@@ -39,6 +40,8 @@ public interface BaseResourceManager {
 
     void putFile(String path, MultipartFile file);
 
+    void putFile(String path, File file);
+
     String getFileContent(String path);
 
     InputStream readFile(String path);

diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/ResourcesService.java b/dinky-admin/src/main/java/org/dinky/service/resource/ResourcesService.java
@@ -27,6 +27,7 @@
 import java.util.List;
 import java.util.function.Function;
 
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 
 import com.baomidou.mybatisplus.extension.service.IService;
@@ -43,6 +44,8 @@ public interface ResourcesService extends IService<Resources> {
      */
     TreeNodeDTO createFolder(Integer pid, String fileName, String desc);
 
+    TreeNodeDTO createFolderOrGet(Integer pid, String fileName, String desc);
+
     /**
      * Rename an existing folder with the specified parameters.
      *
@@ -86,6 +89,9 @@ public interface ResourcesService extends IService<Resources> {
      */
     File getFile(Integer id);
 
+    @Transactional(rollbackFor = Exception.class)
+    void uploadFile(Integer pid, String desc, File file);
+
     /**
      * Upload a file to the specified folder.
      *

diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/HdfsResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/HdfsResourceManager.java
@@ -26,11 +26,13 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 
 import org.springframework.web.multipart.MultipartFile;
 
+import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
 
 public class HdfsResourceManager implements BaseResourceManager {
@@ -66,6 +68,18 @@ public void putFile(String path, MultipartFile file) {
         }
     }
 
+    @Override
+    public void putFile(String path, File file) {
+        try {
+            FSDataOutputStream stream = getHdfs().create(new Path(getFilePath(path)), true);
+            stream.write(FileUtil.readBytes(file));
+            stream.flush();
+            stream.close();
+        } catch (IOException e) {
+            throw BusException.valueOf("file.upload.failed", e);
+        }
+    }
+
     @Override
     public String getFileContent(String path) {
         return IoUtil.readUtf8(readFile(path));

diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java
@@ -24,13 +24,15 @@
 import org.dinky.service.resource.BaseResourceManager;
 import org.dinky.utils.OssTemplate;
 
+import java.io.File;
 import java.io.InputStream;
 
 import org.springframework.web.multipart.MultipartFile;
 
 import com.amazonaws.services.s3.model.CopyObjectRequest;
 import com.amazonaws.services.s3.model.DeleteObjectRequest;
 
+import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
 
 public class OssResourceManager implements BaseResourceManager {
@@ -63,6 +65,16 @@ public void putFile(String path, MultipartFile file) {
         }
     }
 
+    @Override
+    public void putFile(String path, File file) {
+        try {
+            getOssTemplate()
+                    .putObject(getOssTemplate().getBucketName(), getFilePath(path), FileUtil.getInputStream(file));
+        } catch (Exception e) {
+            throw new DinkyException(e);
+        }
+    }
+
     @Override
     public String getFileContent(String path) {
         return IoUtil.readUtf8(readFile(path));

diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java
@@ -33,6 +33,7 @@
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -75,6 +76,30 @@ public TreeNodeDTO createFolder(Integer pid, String fileName, String desc) {
         return convertTree(resources);
     }
 
+    @Override
+    public TreeNodeDTO createFolderOrGet(Integer pid, String fileName, String desc) {
+        String path = "/" + fileName;
+        Resources resources;
+        long count = count(
+                new LambdaQueryWrapper<Resources>().eq(Resources::getPid, pid).eq(Resources::getFileName, fileName));
+        if (count > 0) {
+            resources = getOne(new LambdaQueryWrapper<Resources>()
+                    .eq(Resources::getPid, pid)
+                    .eq(Resources::getFileName, fileName));
+        } else {
+            resources = new Resources();
+            resources.setPid(pid);
+            resources.setFileName(fileName);
+            resources.setIsDirectory(true);
+            resources.setType(0);
+            resources.setFullName(pid < 1 ? path : getById(pid).getFullName() + path);
+            resources.setSize(0L);
+            resources.setDescription(desc);
+            save(resources);
+        }
+        return convertTree(resources);
+    }
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     public void rename(Integer id, String fileName, String desc) {
@@ -174,15 +199,29 @@ public File getFile(Integer id) {
 
     @Transactional(rollbackFor = Exception.class)
     @Override
-    public void uploadFile(Integer pid, String desc, MultipartFile file) {
+    public void uploadFile(Integer pid, String desc, File file) {
         Resources pResource = RESOURCES_CACHE.get(pid, () -> getById(pid));
         if (!pResource.getIsDirectory()) {
             RESOURCES_CACHE.remove(pid);
             Integer realPid = pResource.getPid();
             pResource = RESOURCES_CACHE.get(pid, () -> getById(realPid));
         }
-        long size = file.getSize();
-        String fileName = file.getOriginalFilename();
+        long size = file.length();
+        String fileName = file.getName();
+        upload(pid, desc, (fullName) -> getBaseResourceManager().putFile(fullName, file), fileName, pResource, size);
+    }
+
+    /**
+     *
+     * @param pid pid
+     * @param desc desc
+     * @param uploadAction uploadAction
+     * @param fileName fileName
+     * @param pResource pResource
+     * @param size size
+     */
+    private void upload(
+            Integer pid, String desc, Consumer<String> uploadAction, String fileName, Resources pResource, long size) {
         Resources currentUploadResource = getOne(
                 new LambdaQueryWrapper<Resources>().eq(Resources::getPid, pid).eq(Resources::getFileName, fileName));
         String fullName;
@@ -205,13 +244,27 @@ public void uploadFile(Integer pid, String desc, MultipartFile file) {
             resources.setDescription(desc);
             saveOrUpdate(resources);
         }
-        getBaseResourceManager().putFile(fullName, file);
+        uploadAction.accept(fullName);
 
         List<Resources> resourceByPidToParent = getResourceByPidToParent(new ArrayList<>(), pid);
         resourceByPidToParent.forEach(x -> x.setSize(x.getSize() + size));
         updateBatchById(resourceByPidToParent);
     }
 
+    @Transactional(rollbackFor = Exception.class)
+    @Override
+    public void uploadFile(Integer pid, String desc, MultipartFile file) {
+        Resources pResource = RESOURCES_CACHE.get(pid, () -> getById(pid));
+        if (!pResource.getIsDirectory()) {
+            RESOURCES_CACHE.remove(pid);
+            Integer realPid = pResource.getPid();
+            pResource = RESOURCES_CACHE.get(pid, () -> getById(realPid));
+        }
+        long size = file.getSize();
+        String fileName = file.getOriginalFilename();
+        upload(pid, desc, (fullName) -> getBaseResourceManager().putFile(fullName, file), fileName, pResource, size);
+    }
+
     @Transactional(rollbackFor = Exception.class)
     @Override
     public boolean remove(Integer id) {
@@ -306,6 +359,7 @@ public List<Resources> getResourcesTree() {
      * @param filterFunction filter function
      * @return {@link Result}< {@link List}< {@link Resources}>>}
      */
+    @Override
     public List<Resources> getResourcesTreeByFilter(Function<Resources, Boolean> filterFunction) {
         List<Resources> list = this.list();
         return buildResourcesTree(
@@ -388,7 +442,7 @@ private List<Resources> getChildList(List<Resources> list, Resources resources)
      * @return
      */
     private boolean hasChild(List<Resources> resourcesList, Resources resources) {
-        return getChildList(resourcesList, resources).size() > 0;
+        return !getChildList(resourcesList, resources).isEmpty();
     }
 
     private BaseResourceManager getBaseResourceManager() {

diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfClassStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfClassStepSse.java
@@ -24,8 +24,8 @@
 import org.dinky.data.model.GitProject;
 import org.dinky.function.util.UDFUtil;
 import org.dinky.sse.StepSse;
+import org.dinky.utils.URLUtils;
 
-import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -70,7 +70,7 @@ public void exec() {
             throw new DinkyException("flink dependency not found");
         }
         pathList.parallelStream().forEach(jar -> {
-            List<Class<?>> udfClassByJar = UDFUtil.getUdfClassByJar(new File(jar));
+            List<Class<?>> udfClassByJar = UDFUtil.getUdfClassByJar(URLUtils.toFile(jar));
             udfMap.put(jar, udfClassByJar);
             sendMsg(Dict.create().set(jar, udfClassByJar));
         });

diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfPythonStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/AnalysisUdfPythonStepSse.java
@@ -53,16 +53,16 @@ public AnalysisUdfPythonStepSse(
     @Override
     public void exec() {
         File zipFile = (File) params.get("zipFile");
-        File projectFile = (File) params.get("projectFile");
+        String zipFilePath = params.getStr("zipFilePath");
         try {
             Thread.currentThread().getContextClassLoader().loadClass("org.apache.flink.table.api.ValidationException");
         } catch (ClassNotFoundException e) {
             throw new DinkyException("flink dependency not found");
         }
-        List<String> pythonUdfList = UDFUtil.getPythonUdfList(
-                SystemConfiguration.getInstances().getPythonHome(), projectFile.getAbsolutePath());
+        List<String> pythonUdfList =
+                UDFUtil.getPythonUdfList(SystemConfiguration.getInstances().getPythonHome(), zipFile.getAbsolutePath());
         GitAnalysisJarDTO gitAnalysisJarDTO = new GitAnalysisJarDTO();
-        gitAnalysisJarDTO.setJarPath(zipFile.getAbsolutePath());
+        gitAnalysisJarDTO.setJarPath(zipFilePath);
         gitAnalysisJarDTO.setClassList(pythonUdfList);
 
         List<GitAnalysisJarDTO> dataList = CollUtil.newArrayList(gitAnalysisJarDTO);

diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java
@@ -19,6 +19,9 @@
 
 package org.dinky.sse.git;
 
+import org.dinky.data.dto.TreeNodeDTO;
+import org.dinky.data.model.GitProject;
+import org.dinky.service.resource.ResourcesService;
 import org.dinky.sse.StepSse;
 import org.dinky.utils.MavenUtil;
 
@@ -30,7 +33,9 @@
 
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
 
+import cn.hutool.core.convert.Convert;
 import cn.hutool.core.lang.Dict;
+import cn.hutool.extra.spring.SpringUtil;
 
 /**
  * @author ZackYoung
@@ -51,10 +56,22 @@ public GetJarsStepSse(
     @Override
     public void exec() {
         List<File> jars = MavenUtil.getJars((File) params.get("pom"));
-
-        List<String> pathList = jars.stream().map(File::getAbsolutePath).collect(Collectors.toList());
+        List<String> pathList = uploadResources(jars);
         addFileMsg(pathList);
 
         params.put("jarPath", pathList);
     }
+
+    private List<String> uploadResources(List<File> jars) {
+        GitProject gitProject = (GitProject) params.get("gitProject");
+
+        ResourcesService resourcesService = SpringUtil.getBean(ResourcesService.class);
+        TreeNodeDTO gitFolder = resourcesService.createFolderOrGet(0, "git", "");
+        TreeNodeDTO treeNodeDTO =
+                resourcesService.createFolderOrGet(Convert.toInt(gitFolder.getId()), gitProject.getName(), "");
+        return jars.stream()
+                .peek(f -> resourcesService.uploadFile(Convert.toInt(treeNodeDTO.getId()), "", f))
+                .map(x -> "rs:/git/" + gitProject.getName() + "/" + x.getName())
+                .collect(Collectors.toList());
+    }
 }