From d64de82913b819c2b81b6d24efa7503ae484f332 Mon Sep 17 00:00:00 2001 From: Wink <32723967+aiwenmo@users.noreply.github.com> Date: Wed, 27 Nov 2024 22:49:29 +0800 Subject: [PATCH] [Optimization-3959][core] Optimize submit task print error log (#3960) --- dinky-admin/pom.xml | 16 +++++ .../main/java/org/dinky/init/SystemInit.java | 6 ++ .../factory/CatalogueTreeSortFactory.java | 1 - .../catalogue/impl/CatalogueServiceImpl.java | 1 - .../dinky/service/impl/UDFServiceImpl.java | 4 +- dinky-app/dinky-app-1.20/pom.xml | 1 + dinky-app/dinky-app-base/pom.xml | 12 ---- dinky-assembly/src/main/assembly/package.xml | 8 +++ dinky-core/pom.xml | 39 ------------ .../dinky/data/result/MockResultRunnable.java | 8 +-- .../java/org/dinky/explainer/Explainer.java | 4 +- .../main/java/org/dinky/job/JobManager.java | 13 ++-- .../org/dinky/job/runner/JobDDLRunner.java | 7 +-- .../dinky/job/runner/JobPipelineRunner.java | 7 +-- .../org/dinky/job/runner/JobSetRunner.java | 7 +-- .../org/dinky/job/runner/JobSqlRunner.java | 21 +++---- .../main/java/org/dinky/trans/Operations.java | 29 --------- dinky-function/pom.xml | 5 +- .../org/dinky/function/FlinkUDFDiscover.java | 59 +++++++++++++++++++ dinky-gateway/pom.xml | 4 -- dinky-metadata/dinky-metadata-paimon/pom.xml | 2 + pom.xml | 2 +- 22 files changed, 125 insertions(+), 131 deletions(-) create mode 100644 dinky-function/src/main/java/org/dinky/function/FlinkUDFDiscover.java diff --git a/dinky-admin/pom.xml b/dinky-admin/pom.xml index 0621004116..8181ec7da1 100644 --- a/dinky-admin/pom.xml +++ b/dinky-admin/pom.xml @@ -359,10 +359,26 @@ org.dinky dinky-alert-dingtalk + + org.dinky + dinky-alert-feishu + + + org.dinky + dinky-alert-wechat + org.dinky dinky-alert-sms + + org.dinky + dinky-alert-email + + + org.dinky + dinky-alert-http + diff --git a/dinky-admin/src/main/java/org/dinky/init/SystemInit.java b/dinky-admin/src/main/java/org/dinky/init/SystemInit.java index bc1af97dd4..a2c04c7840 100644 --- a/dinky-admin/src/main/java/org/dinky/init/SystemInit.java +++ b/dinky-admin/src/main/java/org/dinky/init/SystemInit.java @@ -31,6 +31,7 @@ import org.dinky.data.model.Task; import org.dinky.data.model.job.JobInstance; import org.dinky.data.model.rbac.Tenant; +import org.dinky.function.FlinkUDFDiscover; import org.dinky.function.constant.PathConstant; import org.dinky.function.pool.UdfCodePool; import org.dinky.job.ClearJobHistoryTask; @@ -106,6 +107,7 @@ public void run(ApplicationArguments args) { initDaemon(); initDolphinScheduler(); registerUDF(); + discoverUDF(); updateGitBuildState(); registerURL(); } catch (NoClassDefFoundError e) { @@ -220,6 +222,10 @@ public void registerUDF() { UdfCodePool.updateGitPool(gitProjectService.getGitPool()); } + public void discoverUDF() { + FlinkUDFDiscover.getCustomStaticUDFs(); + } + public void updateGitBuildState() { String path = PathConstant.TMP_PATH + "/build.list"; if (FileUtil.exist(path)) { diff --git a/dinky-admin/src/main/java/org/dinky/service/catalogue/factory/CatalogueTreeSortFactory.java b/dinky-admin/src/main/java/org/dinky/service/catalogue/factory/CatalogueTreeSortFactory.java index 393a05f1b3..b3b0de57ba 100644 --- a/dinky-admin/src/main/java/org/dinky/service/catalogue/factory/CatalogueTreeSortFactory.java +++ b/dinky-admin/src/main/java/org/dinky/service/catalogue/factory/CatalogueTreeSortFactory.java @@ -55,7 +55,6 @@ public CatalogueTreeSortStrategy getStrategy(String strategyName) { CatalogueTreeSortStrategy catalogueTreeSortStrategy = Safes.of(catalogueTreeSortStrategyMap).get(strategyName); if (Objects.isNull(catalogueTreeSortStrategy)) { - log.warn("Strategy {} is not defined. Use DefaultStrategy", strategyName); catalogueTreeSortStrategy = Safes.of(catalogueTreeSortStrategyMap).get(CatalogueSortConstant.STRATEGY_DEFAULT); } diff --git a/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java index 1f8f4e63b0..392bf968c4 100644 --- a/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java @@ -121,7 +121,6 @@ public class CatalogueServiceImpl extends SuperServiceImpl getCatalogueTree(CatalogueTreeQueryDTO catalogueTreeQueryDto) { - log.info("getCatalogueTree, catalogueTreeQueryDto: {}", catalogueTreeQueryDto); List catalogueTree = buildCatalogueTree(this.list()); // sort CatalogueTreeSortStrategy strategy = catalogueTreeSortFactory.getStrategy(catalogueTreeQueryDto.getSortValue()); diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/UDFServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/UDFServiceImpl.java index 4819893d68..3c802b1904 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/UDFServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/UDFServiceImpl.java @@ -24,11 +24,11 @@ import org.dinky.data.model.udf.UDFManage; import org.dinky.data.vo.CascaderVO; import org.dinky.data.vo.UDFManageVO; +import org.dinky.function.FlinkUDFDiscover; import org.dinky.function.data.model.UDF; import org.dinky.mapper.UDFManageMapper; import org.dinky.service.UDFService; import org.dinky.service.resource.ResourcesService; -import org.dinky.trans.Operations; import org.dinky.utils.UDFUtils; import org.apache.flink.table.catalog.FunctionLanguage; @@ -182,7 +182,7 @@ public List getUDFFromUdfManage() { @Override public List getAllUdfsToCascader(List userDefinedReleaseUdfs) { // Get all UDFs of static UDFs and dynamic UDFs - List staticUdfs = Operations.getCustomStaticUdfs(); + List staticUdfs = FlinkUDFDiscover.getCustomStaticUDFs(); // get all UDFs of UDFManage table List udfManageDynamic = getUDFFromUdfManage().stream() diff --git a/dinky-app/dinky-app-1.20/pom.xml b/dinky-app/dinky-app-1.20/pom.xml index 8b69fc7600..d1ac0e0879 100644 --- a/dinky-app/dinky-app-1.20/pom.xml +++ b/dinky-app/dinky-app-1.20/pom.xml @@ -60,6 +60,7 @@ ${project.parent.parent.basedir}/build/extends + diff --git a/dinky-app/dinky-app-base/pom.xml b/dinky-app/dinky-app-base/pom.xml index 405114bfdd..c4cad0dd62 100644 --- a/dinky-app/dinky-app-base/pom.xml +++ b/dinky-app/dinky-app-base/pom.xml @@ -46,10 +46,6 @@ org.dinky dinky-core - - com.github.xiaoymin - knife4j-openapi2-spring-boot-starter - com.github.docker-java docker-java-core @@ -82,14 +78,6 @@ org.dinky dinky-gateway - - org.dinky - dinky-alert-sms - - - org.dinky - dinky-alert-dingtalk - diff --git a/dinky-assembly/src/main/assembly/package.xml b/dinky-assembly/src/main/assembly/package.xml index cabb063d48..7cb907dc29 100644 --- a/dinky-assembly/src/main/assembly/package.xml +++ b/dinky-assembly/src/main/assembly/package.xml @@ -122,6 +122,14 @@ dinky-client-1.19-${project.version}.jar + + ${project.parent.basedir}/build/extends/ + extends/flink1.20/dinky + + dinky-catalog-mysql-1.20-${project.version}.jar + dinky-client-1.20-${project.version}.jar + + ${project.parent.basedir}/build/extends/ extends diff --git a/dinky-core/pom.xml b/dinky-core/pom.xml index 819603f2d5..ce2d170444 100644 --- a/dinky-core/pom.xml +++ b/dinky-core/pom.xml @@ -32,11 +32,6 @@ 3.1.0 - org.dinky dinky-common @@ -54,36 +49,6 @@ dinky-metadata-base ${scope.runtime} - - org.dinky - dinky-alert-dingtalk - ${scope.runtime} - - - org.dinky - dinky-alert-wechat - ${scope.runtime} - - - org.dinky - dinky-alert-feishu - ${scope.runtime} - - - org.dinky - dinky-alert-sms - ${scope.runtime} - - - org.dinky - dinky-alert-email - ${scope.runtime} - - - org.dinky - dinky-alert-http - ${scope.runtime} - org.dinky dinky-metadata-paimon @@ -203,10 +168,6 @@ junit-jupiter test - - org.reflections - reflections - org.slf4j slf4j-log4j12 diff --git a/dinky-core/src/main/java/org/dinky/data/result/MockResultRunnable.java b/dinky-core/src/main/java/org/dinky/data/result/MockResultRunnable.java index 3f5cd041fd..28dc9ddda0 100644 --- a/dinky-core/src/main/java/org/dinky/data/result/MockResultRunnable.java +++ b/dinky-core/src/main/java/org/dinky/data/result/MockResultRunnable.java @@ -122,7 +122,7 @@ private void catchChangLog(SelectResult selectResult) { for (String tableIdentifier : tableIdentifierList) { if (!tableIdentifierIndexMap.containsKey(tableIdentifier)) { tableIdentifierIndexMap.put(tableIdentifier, 0); - } else if (tableIdentifierIndexMap.get(tableIdentifier) >= maxRowNum) { + } else if (tableIdentifierIndexMap.get(tableIdentifier) >= maxRowNum - 1) { allSinkFinished = true; continue; } @@ -145,7 +145,7 @@ private void catchChangLog(SelectResult selectResult) { rows.add(rowDataWithTableIdentifier); tableIdentifierIndexMap.put( tableIdentifier, tableIdentifierIndexMap.get(tableIdentifier) + 1); - if (tableIdentifierIndexMap.get(tableIdentifier) > maxRowNum) { + if (tableIdentifierIndexMap.get(tableIdentifier) >= maxRowNum) { break; } } @@ -190,7 +190,7 @@ private void catchData(SelectResult selectResult) { for (String tableIdentifier : tableIdentifierList) { if (!tableIdentifierIndexMap.containsKey(tableIdentifier)) { tableIdentifierIndexMap.put(tableIdentifier, 0); - } else if (tableIdentifierIndexMap.get(tableIdentifier) >= maxRowNum) { + } else if (tableIdentifierIndexMap.get(tableIdentifier) >= maxRowNum - 1) { allSinkFinished = true; continue; } @@ -219,7 +219,7 @@ private void catchData(SelectResult selectResult) { } tableIdentifierIndexMap.put( tableIdentifier, tableIdentifierIndexMap.get(tableIdentifier) + 1); - if (tableIdentifierIndexMap.get(tableIdentifier) > maxRowNum) { + if (tableIdentifierIndexMap.get(tableIdentifier) >= maxRowNum) { break; } } diff --git a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java index 96bb6c6445..4fed7f4afd 100644 --- a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java +++ b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java @@ -41,6 +41,7 @@ import org.dinky.job.builder.JobUDFBuilder; import org.dinky.trans.Operations; import org.dinky.utils.DinkyClassLoaderUtil; +import org.dinky.utils.LogUtil; import org.dinky.utils.SqlUtil; import org.apache.flink.runtime.rest.messages.JobPlanInfo; @@ -141,8 +142,9 @@ public ExplainResult explainSql(String statement) { jobStatementPlan.buildFinalStatement(); jobManager.setJobStatementPlan(jobStatementPlan); } catch (Exception e) { + String error = LogUtil.getError("Exception in parsing FlinkSQL:\n" + SqlUtil.addLineNumber(statement), e); SqlExplainResult.Builder resultBuilder = SqlExplainResult.Builder.newBuilder(); - resultBuilder.error(e.getMessage()).parseTrue(false); + resultBuilder.error(error).parseTrue(false); sqlExplainRecords.add(resultBuilder.build()); log.error("Failed parseStatements:", e); return new ExplainResult(false, sqlExplainRecords.size(), sqlExplainRecords); diff --git a/dinky-core/src/main/java/org/dinky/job/JobManager.java b/dinky-core/src/main/java/org/dinky/job/JobManager.java index 4c4de9a488..f25fa56dd0 100644 --- a/dinky-core/src/main/java/org/dinky/job/JobManager.java +++ b/dinky-core/src/main/java/org/dinky/job/JobManager.java @@ -92,7 +92,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.text.StrFormatter; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -315,16 +314,14 @@ public JobResult executeSql(String statement) throws Exception { } } catch (Exception e) { String errorMessage = e.getMessage(); - if (errorMessage != null && errorMessage.contains("Only insert statement is supported now")) { - throw new BusException(Status.OPERATE_NOT_SUPPORT_QUERY.getMessage()); - } - String error = StrFormatter.format( - "Exception in executing FlinkSQL:\n{}\n{}", SqlUtil.addLineNumber(currentSql), errorMessage); job.setEndTime(LocalDateTime.now()); job.setStatus(Job.JobStatus.FAILED); - job.setError(error); + job.setError(errorMessage); failed(); - throw new Exception(error, e); + if (errorMessage != null && errorMessage.contains("Only insert statement is supported now")) { + throw new BusException(Status.OPERATE_NOT_SUPPORT_QUERY.getMessage()); + } + throw new Exception(errorMessage, e); } finally { close(); } diff --git a/dinky-core/src/main/java/org/dinky/job/runner/JobDDLRunner.java b/dinky-core/src/main/java/org/dinky/job/runner/JobDDLRunner.java index c46fff071a..59bb856ee8 100644 --- a/dinky-core/src/main/java/org/dinky/job/runner/JobDDLRunner.java +++ b/dinky-core/src/main/java/org/dinky/job/runner/JobDDLRunner.java @@ -47,7 +47,6 @@ import java.util.Set; import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.text.StrFormatter; import cn.hutool.core.util.ArrayUtil; import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; @@ -125,10 +124,8 @@ public SqlExplainResult explain(JobStatement jobStatement) { .sql(jobStatement.getStatement()) .index(jobStatement.getIndex()); } catch (Exception e) { - String error = StrFormatter.format( - "Exception in explaining FlinkSQL:\n{}\n{}", - SqlUtil.addLineNumber(jobStatement.getStatement()), - LogUtil.getError(e)); + String error = LogUtil.getError( + "Exception in explaining FlinkSQL:\n" + SqlUtil.addLineNumber(jobStatement.getStatement()), e); resultBuilder .error(error) .explainTrue(false) diff --git a/dinky-core/src/main/java/org/dinky/job/runner/JobPipelineRunner.java b/dinky-core/src/main/java/org/dinky/job/runner/JobPipelineRunner.java index 07cae22b8f..8c7c1f18e9 100644 --- a/dinky-core/src/main/java/org/dinky/job/runner/JobPipelineRunner.java +++ b/dinky-core/src/main/java/org/dinky/job/runner/JobPipelineRunner.java @@ -51,7 +51,6 @@ import java.util.ArrayList; import java.util.List; -import cn.hutool.core.text.StrFormatter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -100,10 +99,8 @@ public SqlExplainResult explain(JobStatement jobStatement) { .explainTime(LocalDateTime.now()) .index(jobStatement.getIndex()); } catch (Exception e) { - String error = StrFormatter.format( - "Exception in explaining FlinkSQL:\n{}\n{}", - SqlUtil.addLineNumber(jobStatement.getStatement()), - LogUtil.getError(e)); + String error = LogUtil.getError( + "Exception in explaining FlinkSQL:\n" + SqlUtil.addLineNumber(jobStatement.getStatement()), e); resultBuilder .parseTrue(false) .error(error) diff --git a/dinky-core/src/main/java/org/dinky/job/runner/JobSetRunner.java b/dinky-core/src/main/java/org/dinky/job/runner/JobSetRunner.java index 277738cdc9..03e95f158a 100644 --- a/dinky-core/src/main/java/org/dinky/job/runner/JobSetRunner.java +++ b/dinky-core/src/main/java/org/dinky/job/runner/JobSetRunner.java @@ -29,7 +29,6 @@ import java.time.LocalDateTime; -import cn.hutool.core.text.StrFormatter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -66,10 +65,8 @@ public SqlExplainResult explain(JobStatement jobStatement) { .explainTime(LocalDateTime.now()) .index(jobStatement.getIndex()); } catch (Exception e) { - String error = StrFormatter.format( - "Exception in explaining FlinkSQL:\n{}\n{}", - SqlUtil.addLineNumber(jobStatement.getStatement()), - LogUtil.getError(e)); + String error = LogUtil.getError( + "Exception in explaining FlinkSQL:\n" + SqlUtil.addLineNumber(jobStatement.getStatement()), e); resultBuilder .parseTrue(false) .error(error) diff --git a/dinky-core/src/main/java/org/dinky/job/runner/JobSqlRunner.java b/dinky-core/src/main/java/org/dinky/job/runner/JobSqlRunner.java index 9b2f1be2ee..f0d4bd268b 100644 --- a/dinky-core/src/main/java/org/dinky/job/runner/JobSqlRunner.java +++ b/dinky-core/src/main/java/org/dinky/job/runner/JobSqlRunner.java @@ -37,6 +37,7 @@ import org.dinky.job.Job; import org.dinky.job.JobConfig; import org.dinky.job.JobManager; +import org.dinky.utils.LogUtil; import org.dinky.utils.SqlUtil; import org.dinky.utils.URLUtils; @@ -54,7 +55,6 @@ import java.util.UUID; import java.util.stream.Collectors; -import cn.hutool.core.text.StrFormatter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -89,10 +89,8 @@ public SqlExplainResult explain(JobStatement jobStatement) { jobManager.getExecutor().explainSqlRecord(jobStatement.getStatement())); resultBuilder.parseTrue(true).explainTrue(true); } catch (Exception e) { - String error = StrFormatter.format( - "Exception in explaining FlinkSQL:\n{}\n{}", - SqlUtil.addLineNumber(jobStatement.getStatement()), - e.getMessage()); + String error = LogUtil.getError( + "Exception in explaining FlinkSQL:\n" + SqlUtil.addLineNumber(jobStatement.getStatement()), e); resultBuilder .type(jobStatement.getSqlType().getType()) .index(jobStatement.getIndex()) @@ -123,10 +121,9 @@ public SqlExplainResult explain(JobStatement jobStatement) { resultBuilder = SqlExplainResult.newBuilder(jobManager.getExecutor().explainStatementSet(statements)); } catch (Exception e) { - String error = StrFormatter.format( - "Exception in explaining FlinkSQL:\n{}\n{}", - SqlUtil.addLineNumber(jobStatement.getStatement()), - e.getMessage()); + String error = LogUtil.getError( + "Exception in explaining FlinkSQL:\n" + SqlUtil.addLineNumber(jobStatement.getStatement()), + e); resultBuilder .sql(sqlSet) .index(jobStatement.getIndex()) @@ -152,10 +149,8 @@ public SqlExplainResult explain(JobStatement jobStatement) { jobManager.getExecutor().explainSqlRecord(jobStatement.getStatement())); resultBuilder.parseTrue(true).explainTrue(true); } catch (Exception e) { - String error = StrFormatter.format( - "Exception in explaining FlinkSQL:\n{}\n{}", - SqlUtil.addLineNumber(jobStatement.getStatement()), - e.getMessage()); + String error = LogUtil.getError( + "Exception in explaining FlinkSQL:\n" + SqlUtil.addLineNumber(jobStatement.getStatement()), e); resultBuilder .type(jobStatement.getSqlType().getType()) .index(jobStatement.getIndex()) diff --git a/dinky-core/src/main/java/org/dinky/trans/Operations.java b/dinky-core/src/main/java/org/dinky/trans/Operations.java index 76705eda2e..0bd5d362c3 100644 --- a/dinky-core/src/main/java/org/dinky/trans/Operations.java +++ b/dinky-core/src/main/java/org/dinky/trans/Operations.java @@ -20,23 +20,14 @@ package org.dinky.trans; import org.dinky.data.job.SqlType; -import org.dinky.function.data.model.UDF; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.flink.table.catalog.FunctionLanguage; -import org.apache.flink.table.functions.UserDefinedFunction; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; -import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import org.reflections.Reflections; import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; import lombok.extern.slf4j.Slf4j; @@ -54,7 +45,6 @@ private Operations() {} private static final Operation[] ALL_OPERATIONS = getAllOperations(); - private static final List JAVA_STATIC_UDF_LIST = getCustomStaticUdfs(); /** * get all {@link Operation} children ordinary class, * @@ -105,23 +95,4 @@ public static Operation buildOperation(String statement) { .map(p -> p.create(statement)) .orElse(null); } - - public static List getCustomStaticUdfs() { - if (CollectionUtils.isNotEmpty(JAVA_STATIC_UDF_LIST)) { - return JAVA_STATIC_UDF_LIST; - } - - Reflections reflections = - new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forJavaClassPath())); - Set> operations = - reflections.get(Scanners.SubTypes.of(UserDefinedFunction.class).asClass()); - return operations.stream() - .filter(operation -> - !operation.isInterface() && !operation.getName().startsWith("org.apache")) - .map(operation -> UDF.builder() - .className(operation.getName()) - .functionLanguage(FunctionLanguage.JAVA) - .build()) - .collect(Collectors.toList()); - } } diff --git a/dinky-function/pom.xml b/dinky-function/pom.xml index 5d542e360f..8afce1f9a8 100644 --- a/dinky-function/pom.xml +++ b/dinky-function/pom.xml @@ -74,7 +74,10 @@ lombok provided - + + org.reflections + reflections + org.junit.vintage diff --git a/dinky-function/src/main/java/org/dinky/function/FlinkUDFDiscover.java b/dinky-function/src/main/java/org/dinky/function/FlinkUDFDiscover.java new file mode 100644 index 0000000000..2b9ab4f595 --- /dev/null +++ b/dinky-function/src/main/java/org/dinky/function/FlinkUDFDiscover.java @@ -0,0 +1,59 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dinky.function; + +import org.dinky.function.data.model.UDF; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.flink.table.catalog.FunctionLanguage; +import org.apache.flink.table.functions.UserDefinedFunction; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; + +public class FlinkUDFDiscover { + + private static final List JAVA_STATIC_UDF_LIST = getCustomStaticUDFs(); + + public static List getCustomStaticUDFs() { + if (CollectionUtils.isNotEmpty(JAVA_STATIC_UDF_LIST)) { + return JAVA_STATIC_UDF_LIST; + } + + Reflections reflections = + new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forJavaClassPath())); + Set> operations = + reflections.get(Scanners.SubTypes.of(UserDefinedFunction.class).asClass()); + return operations.stream() + .filter(operation -> + !operation.isInterface() && !operation.getName().startsWith("org.apache")) + .map(operation -> UDF.builder() + .className(operation.getName()) + .functionLanguage(FunctionLanguage.JAVA) + .build()) + .collect(Collectors.toList()); + } +} diff --git a/dinky-gateway/pom.xml b/dinky-gateway/pom.xml index eaf17b63c5..e60a16f066 100644 --- a/dinky-gateway/pom.xml +++ b/dinky-gateway/pom.xml @@ -31,10 +31,6 @@ Dinky : Gateway - - com.github.xiaoymin - knife4j-openapi2-spring-boot-starter - io.fabric8 kubernetes-client diff --git a/dinky-metadata/dinky-metadata-paimon/pom.xml b/dinky-metadata/dinky-metadata-paimon/pom.xml index 2aa1d5e6c3..afa612b728 100644 --- a/dinky-metadata/dinky-metadata-paimon/pom.xml +++ b/dinky-metadata/dinky-metadata-paimon/pom.xml @@ -40,12 +40,14 @@ org.apache.paimon paimon-bundle 0.8.1 + ${scope.runtime} org.apache.paimon paimon-s3 0.8.1 + ${scope.runtime} diff --git a/pom.xml b/pom.xml index 530f560fb2..00ceeb9c00 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ UTF-8 2.5.0 0.10.2 - 1.2.0 + 1.2.0-rc2 1.37.0 2.12.10