[Optimize] You can submit Yarn-Application (Linux) on Windows (#2541)

* fix_app * fix_web_some_bug * fix_app * fix_app * fix_app * fix_app_some_bug * fix_app_some_bug * fix_app_some_bug
DataLinkDC · Nov 17, 2023 · b855080 · b855080
1 parent 4f6c1c1
commit b855080
Show file tree

Hide file tree

Showing 59 changed files with 12,702 additions and 603 deletions.
diff --git a/dinky-admin/pom.xml b/dinky-admin/pom.xml
@@ -44,11 +44,6 @@
             <groupId>org.jasypt</groupId>
             <artifactId>jasypt</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-s3</artifactId>
-            <version>1.12.481</version>
-        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>

diff --git a/dinky-admin/src/main/java/org/dinky/controller/DownloadController.java b/dinky-admin/src/main/java/org/dinky/controller/DownloadController.java
@@ -26,6 +26,7 @@
 import org.dinky.data.model.FlinkUdfManifest;
 import org.dinky.function.constant.PathConstant;
 import org.dinky.function.util.ZipWriter;
+import org.dinky.service.resource.BaseResourceManager;
 
 import java.io.File;
 import java.io.InputStream;
@@ -50,7 +51,9 @@
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
 
-/** @since 0.7.0 */
+/**
+ * @since 0.7.0
+ */
 @Slf4j
 @RestController
 @Api(tags = "UDF & App Jar Controller")
@@ -102,7 +105,7 @@ public void downloadJavaUDF(@PathVariable Integer taskId, HttpServletResponse re
      * 提供docker通过http下载dinky-app.jar
      *
      * @param version 版本
-     * @param resp resp
+     * @param resp    resp
      */
     @GetMapping("downloadAppJar/{version}")
     @Log(title = "Download App Jar", businessType = BusinessType.DOWNLOAD)
@@ -114,4 +117,12 @@ public void downloadAppJar(@PathVariable String version, HttpServletResponse res
             ServletUtil.write(resp, files.get(0));
         }
     }
+
+    @GetMapping("downloadFromRs")
+    @Log(title = "Download From Resource", businessType = BusinessType.DOWNLOAD)
+    @ApiOperation("Download From Resource")
+    public void downloadJavaUDF(String path, HttpServletResponse resp) {
+        InputStream inputStream = BaseResourceManager.getInstance().readFile(path);
+        ServletUtil.write(resp, inputStream);
+    }
 }
diff --git a/dinky-admin/src/main/java/org/dinky/init/SystemInit.java b/dinky-admin/src/main/java/org/dinky/init/SystemInit.java
@@ -34,6 +34,7 @@
 import org.dinky.function.constant.PathConstant;
 import org.dinky.function.pool.UdfCodePool;
 import org.dinky.job.FlinkJobTask;
+import org.dinky.oss.OssTemplate;
 import org.dinky.scheduler.client.ProjectClient;
 import org.dinky.scheduler.exception.SchedulerException;
 import org.dinky.scheduler.model.Project;
@@ -46,7 +47,6 @@
 import org.dinky.service.resource.impl.OssResourceManager;
 import org.dinky.url.RsURLStreamHandlerFactory;
 import org.dinky.utils.JsonUtils;
-import org.dinky.utils.OssTemplate;
 import org.dinky.utils.UDFUtils;
 
 import org.apache.catalina.webresources.TomcatURLStreamHandlerFactory;
@@ -98,9 +98,9 @@ public class SystemInit implements ApplicationRunner {
     public void run(ApplicationArguments args) {
         TenantContextHolder.ignoreTenant();
         initResources();
-
         List<Tenant> tenants = tenantService.list();
         sysConfigService.initSysConfig();
+
         for (Tenant tenant : tenants) {
             taskService.initDefaultFlinkSQLEnv(tenant.getId());
         }

diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/OssResourceManager.java
@@ -21,8 +21,8 @@
 
 import org.dinky.data.exception.BusException;
 import org.dinky.data.exception.DinkyException;
+import org.dinky.oss.OssTemplate;
 import org.dinky.service.resource.BaseResourceManager;
-import org.dinky.utils.OssTemplate;
 
 import java.io.File;
 import java.io.InputStream;

diff --git a/dinky-app/dinky-app-1.14/pom.xml b/dinky-app/dinky-app-1.14/pom.xml
@@ -40,10 +40,6 @@
             <artifactId>dinky-app-base</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
             <artifactId>dinky-client-1.14</artifactId>
@@ -53,14 +49,6 @@
             <artifactId>dinky-flink-1.14</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
-        <dependency>
-            <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-base</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.dinky</groupId>
-            <artifactId>dinky-executor</artifactId>
-        </dependency>
     </dependencies>
 
     <build>

diff --git a/dinky-app/dinky-app-1.14/src/main/java/org/dinky/app/MainApp.java b/dinky-app/dinky-app-1.14/src/main/java/org/dinky/app/MainApp.java
@@ -22,7 +22,6 @@
 import org.dinky.app.constant.AppParamConstant;
 import org.dinky.app.db.DBUtil;
 import org.dinky.app.flinksql.Submitter;
-import org.dinky.app.util.FlinkAppUtil;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.utils.JsonUtils;
 
@@ -56,9 +55,6 @@ public static void main(String[] args) throws Exception {
         } catch (Exception e) {
             log.error("exectue app failed with config: {}", appConfig);
             throw e;
-        } finally {
-            log.info("Start Monitor Job");
-            FlinkAppUtil.monitorFlinkTask(appConfig.getTaskId());
         }
     }
 }
diff --git a/dinky-app/dinky-app-1.15/pom.xml b/dinky-app/dinky-app-1.15/pom.xml
@@ -39,10 +39,6 @@
             <groupId>org.dinky</groupId>
             <artifactId>dinky-app-base</artifactId>
         </dependency>
-        <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
             <artifactId>dinky-client-1.15</artifactId>
@@ -52,14 +48,6 @@
             <artifactId>dinky-flink-1.15</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
-        <dependency>
-            <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-base</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.dinky</groupId>
-            <artifactId>dinky-executor</artifactId>
-        </dependency>
     </dependencies>
 
     <build>

diff --git a/dinky-app/dinky-app-1.15/src/main/java/org/dinky/app/MainApp.java b/dinky-app/dinky-app-1.15/src/main/java/org/dinky/app/MainApp.java
@@ -22,7 +22,6 @@
 import org.dinky.app.constant.AppParamConstant;
 import org.dinky.app.db.DBUtil;
 import org.dinky.app.flinksql.Submitter;
-import org.dinky.app.util.FlinkAppUtil;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.utils.JsonUtils;
 
@@ -56,9 +55,6 @@ public static void main(String[] args) throws Exception {
         } catch (Exception e) {
             log.error("exectue app failed with config: {}", appConfig);
             throw e;
-        } finally {
-            log.info("Start Monitor Job");
-            FlinkAppUtil.monitorFlinkTask(appConfig.getTaskId());
         }
     }
 }
diff --git a/dinky-app/dinky-app-1.16/pom.xml b/dinky-app/dinky-app-1.16/pom.xml
@@ -25,12 +25,11 @@
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-${dinky.flink.version}</artifactId>
-            <scope>${scope.runtime}</scope>
+            <artifactId>dinky-client-1.16</artifactId>
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-flink-${dinky.flink.version}</artifactId>
+            <artifactId>dinky-flink-1.16</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
     </dependencies>

diff --git a/dinky-app/dinky-app-1.16/src/main/java/org/dinky/app/MainApp.java b/dinky-app/dinky-app-1.16/src/main/java/org/dinky/app/MainApp.java
@@ -22,7 +22,6 @@
 import org.dinky.app.constant.AppParamConstant;
 import org.dinky.app.db.DBUtil;
 import org.dinky.app.flinksql.Submitter;
-import org.dinky.app.util.FlinkAppUtil;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.utils.JsonUtils;
 
@@ -55,9 +54,6 @@ public static void main(String[] args) throws Exception {
             Submitter.submit(appConfig);
         } catch (Exception e) {
             log.error("exectue app failed : ", e);
-        } finally {
-            log.info("Start Monitor Job");
-            FlinkAppUtil.monitorFlinkTask(appConfig.getTaskId());
         }
     }
 }
diff --git a/dinky-app/dinky-app-1.17/pom.xml b/dinky-app/dinky-app-1.17/pom.xml
@@ -25,12 +25,11 @@
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-${dinky.flink.version}</artifactId>
-            <scope>${scope.runtime}</scope>
+            <artifactId>dinky-client-1.17</artifactId>
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-flink-${dinky.flink.version}</artifactId>
+            <artifactId>dinky-flink-1.17</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
     </dependencies>

diff --git a/dinky-app/dinky-app-1.17/src/main/java/org/dinky/app/MainApp.java b/dinky-app/dinky-app-1.17/src/main/java/org/dinky/app/MainApp.java
@@ -22,7 +22,6 @@
 import org.dinky.app.constant.AppParamConstant;
 import org.dinky.app.db.DBUtil;
 import org.dinky.app.flinksql.Submitter;
-import org.dinky.app.util.FlinkAppUtil;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.utils.JsonUtils;
 
@@ -56,9 +55,6 @@ public static void main(String[] args) throws Exception {
         } catch (Exception e) {
             log.error("exectue app failed with config: {}", appConfig);
             throw e;
-        } finally {
-            log.info("Start Monitor Job");
-            FlinkAppUtil.monitorFlinkTask(appConfig.getTaskId());
         }
     }
 }
diff --git a/dinky-app/dinky-app-1.18/pom.xml b/dinky-app/dinky-app-1.18/pom.xml
@@ -25,12 +25,11 @@
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-${dinky.flink.version}</artifactId>
-            <scope>${scope.runtime}</scope>
+            <artifactId>dinky-client-1.18</artifactId>
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-flink-${dinky.flink.version}</artifactId>
+            <artifactId>dinky-flink-1.18</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
     </dependencies>

diff --git a/dinky-app/dinky-app-1.18/src/main/java/org/dinky/app/MainApp.java b/dinky-app/dinky-app-1.18/src/main/java/org/dinky/app/MainApp.java
@@ -22,7 +22,6 @@
 import org.dinky.app.constant.AppParamConstant;
 import org.dinky.app.db.DBUtil;
 import org.dinky.app.flinksql.Submitter;
-import org.dinky.app.util.FlinkAppUtil;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.utils.JsonUtils;
 
@@ -56,9 +55,6 @@ public static void main(String[] args) throws Exception {
         } catch (Exception e) {
             log.error("exectue app failed with config: {}", appConfig);
             throw e;
-        } finally {
-            log.info("Start Monitor Job");
-            FlinkAppUtil.monitorFlinkTask(appConfig.getTaskId());
         }
     }
 }
diff --git a/dinky-app/dinky-app-base/pom.xml b/dinky-app/dinky-app-base/pom.xml
@@ -31,13 +31,57 @@
     <name>Dinky : App : Base</name>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>3.1.0</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
             <artifactId>dinky-executor</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.github.xiaoymin</groupId>
+                    <artifactId>knife4j-openapi2-spring-boot-starter</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.github.docker-java</groupId>
+                    <artifactId>docker-java-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.github.docker-java</groupId>
+                    <artifactId>docker-java-transport-httpclient5</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.jeasy</groupId>
+                    <artifactId>easy-rules-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.jeasy</groupId>
+                    <artifactId>easy-rules-spel</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jgit</groupId>
+                    <artifactId>org.eclipse.jgit.archive</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jgit</groupId>
+                    <artifactId>org.eclipse.jgit.ssh.jsch</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.github.oshi</groupId>
+                    <artifactId>oshi-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.dinky</groupId>
+                    <artifactId>dinky-gateway</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>

diff --git a/dinky-app/dinky-app-base/src/main/java/org/dinky/app/db/DBUtil.java b/dinky-app/dinky-app-base/src/main/java/org/dinky/app/db/DBUtil.java
@@ -19,6 +19,7 @@
 
 package org.dinky.app.db;
 
+import org.dinky.app.model.SysConfig;
 import org.dinky.data.app.AppDatabase;
 import org.dinky.data.app.AppParamConfig;
 import org.dinky.data.app.AppTask;
@@ -68,14 +69,8 @@ public static String getDbSourceSQLStatement() throws SQLException {
         return sb.toString();
     }
 
-    public static String getSysConfig(String key) throws SQLException {
-        Entity option = Entity.create("dinky_sys_config").set("name", key);
-        List<Entity> entities = db.find(option);
-        if (entities.size() <= 0) {
-            throw new IllegalArgumentException(
-                    StrFormatter.format("The system conifg is not found: {}, please check! ", key));
-        } else {
-            return entities.get(0).getStr("value");
-        }
+    public static List<SysConfig> getSysConfigList() throws SQLException {
+        Entity option = Entity.create("dinky_sys_config");
+        return db.find(option, SysConfig.class);
     }
 }