Merge branch 'apache:dev' into dev

apache · Nov 28, 2024 · a890e2d · a890e2d
2 parents 7012bdd + 8bec503
commit a890e2d
Show file tree

Hide file tree

Showing 112 changed files with 284 additions and 13,505 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -55,12 +55,18 @@ jobs:
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Collect Workflow Telemetry
+        uses: ./.github/actions/workflow-telemetry-action
+        with:
+          comment_on_pr: false
       - run: sudo npm install -g [email protected]
       - run: sudo apt install plocate -y
       # NOTE: Change command from `find . -name "*.md"` to `find . -not -path "*/node_modules/*" -not -path "*/.tox/*" -name "*.md"`
       # if you want to run check locally
       - run: |
-          for file in $(locate "$PWD*/*.md" | grep -v ./deploy/terraform/aws/README.md); do
+          for file in $(locate "$PWD*/*.md" | grep -v ./deploy/terraform/aws/README.md | grep -v ./.github); do
             markdown-link-check -c .dlc.json -q "$file" &
           done
           wait

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -62,6 +62,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+      - name: Collect Workflow Telemetry
+        uses: ./.github/actions/workflow-telemetry-action
+        with:
+          comment_on_pr: false
       - name: Sanity Check
         uses: ./.github/actions/sanity-check
         with:

diff --git a/docs/docs/en/guide/parameter/context.md b/docs/docs/en/guide/parameter/context.md
@@ -14,7 +14,7 @@ DolphinScheduler allows parameter transfer between tasks. Currently, transfer di
 * [SQL](../task/sql.md)
 * [Procedure](../task/stored-procedure.md)
 * [Python](../task/python.md)
-* [SubWorkflow](../task/sub-workflow)
+* [SubWorkflow](../task/sub-workflow.md)
 * [Kubernetes](../task/kubernetes.md)
 
 When defining an upstream node, if there is a need to transmit the result of that node to a dependency related downstream node. You need to set an `OUT` direction parameter to [Custom Parameters] of the [Current Node Settings]. If it is a sub-workflow node, there is no need to set a parameter in [Current Node Settings], but an `OUT` direction parameter needs to be set in the workflow definition of the sub-workflow.

diff --git a/docs/docs/zh/guide/parameter/context.md b/docs/docs/zh/guide/parameter/context.md
@@ -14,7 +14,7 @@ DolphinScheduler 允许在任务间进行参数传递，目前传递方向仅支
 * [SQL](../task/sql.md)
 * [Procedure](../task/stored-procedure.md)
 * [Python](../task/python.md)
-* [SubWorkflow](../task/sub-workflow)
+* [SubWorkflow](../task/sub-workflow.md)
 * [Kubernetes](../task/kubernetes.md)
 
 当定义上游节点时，如果有需要将该节点的结果传递给有依赖关系的下游节点，需要在【当前节点设置】的【自定义参数】设置一个方向是 OUT 的变量。如果是 SubWorkflow 节点无需在【当前节点设置】中设置变量，需要在子流程的工作流定义中设置一个方向是 OUT 的变量。

diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
@@ -28,15 +28,20 @@
     <name>${project.artifactId}</name>
     <dependencies>
         <!-- dolphinscheduler -->
-
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-meter</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-alert-api</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-alert-all</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
@@ -58,6 +63,7 @@
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-common</artifactId>
         </dependency>
+        <!-- dolphinscheduler -->
 
         <dependency>
             <groupId>com.google.guava</groupId>

diff --git a/dolphinscheduler-api-test/pom.xml b/dolphinscheduler-api-test/pom.xml
@@ -48,6 +48,7 @@
         <jackson.version>2.13.2</jackson.version>
         <checkstyle.version>3.1.2</checkstyle.version>
         <junit-pioneer.version>2.2.0</junit-pioneer.version>
+        <http.components.version>4.5.13</http.components.version>
     </properties>
 
     <dependencies>
@@ -74,6 +75,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>${http.components.version}</version>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>org.testcontainers</groupId>

diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
@@ -61,32 +61,60 @@
             <artifactId>dolphinscheduler-meter</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-datasource-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-datasource-all</artifactId>
             <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-task-api</artifactId>
         </dependency>
 
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-task-all</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-ui</artifactId>
         </dependency>
+
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-registry-all</artifactId>
         </dependency>
+
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-scheduler-all</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-storage-api</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-storage-all</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
@@ -100,13 +128,16 @@
         </dependency>
 
         <dependency>
-            <groupId>org.codehaus.janino</groupId>
-            <artifactId>janino</artifactId>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-extract-alert</artifactId>
         </dependency>
+        <!-- dolphinscheduler -->
+
         <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-s3</artifactId>
+            <groupId>org.codehaus.janino</groupId>
+            <artifactId>janino</artifactId>
         </dependency>
+
         <!--springboot-->
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -228,10 +259,18 @@
             <groupId>com.azure.resourcemanager</groupId>
             <artifactId>azure-resourcemanager-datafactory</artifactId>
         </dependency>
+
         <dependency>
-            <groupId>org.apache.dolphinscheduler</groupId>
-            <artifactId>dolphinscheduler-extract-alert</artifactId>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-identity</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>net.java.dev.jna</groupId>
+                    <artifactId>jna-platform</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
+
     </dependencies>
     <build>
         <testResources>

diff --git a/...uler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/...uler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java
@@ -25,8 +25,6 @@
 import org.apache.dolphinscheduler.dao.entity.AuditLog;
 import org.apache.dolphinscheduler.dao.mapper.AuditLogMapper;
 
-import org.apache.parquet.Strings;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -40,6 +38,7 @@
 
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.google.common.base.Strings;
 
 @Service
 @Slf4j

diff --git a/...ler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/...ler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
@@ -30,6 +30,7 @@
 import org.apache.dolphinscheduler.api.utils.RegexUtils;
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.AuthorizationType;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
 import org.apache.dolphinscheduler.dao.entity.Queue;
 import org.apache.dolphinscheduler.dao.entity.Schedule;
 import org.apache.dolphinscheduler.dao.entity.Tenant;
@@ -255,8 +256,9 @@ public void deleteTenantById(User loginUser, int id) throws Exception {
     }
 
     private List<WorkflowInstance> getWorkflowInstancesByTenant(Tenant tenant) {
-        return workflowInstanceMapper.queryByTenantCodeAndStatus(tenant.getTenantCode(),
-                org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES);
+        return workflowInstanceMapper.queryByTenantCodeAndStatus(
+                tenant.getTenantCode(),
+                WorkflowExecutionStatus.getNotTerminalStatus());
     }
 
     /**

diff --git a/...pi/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/...pi/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
@@ -27,6 +27,7 @@
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.AuthorizationType;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
 import org.apache.dolphinscheduler.dao.entity.Schedule;
@@ -66,7 +67,7 @@
 import org.springframework.transaction.annotation.Transactional;
 
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.facebook.presto.jdbc.internal.guava.base.Strings;
+import com.google.common.base.Strings;
 
 @Service
 @Slf4j
@@ -391,9 +392,9 @@ public Map<String, Object> deleteWorkerGroupById(User loginUser, Integer id) {
             putMsg(result, Status.DELETE_WORKER_GROUP_NOT_EXIST);
             return result;
         }
-        List<WorkflowInstance> workflowInstances = workflowInstanceMapper
-                .queryByWorkerGroupNameAndStatus(workerGroup.getName(),
-                        org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES);
+        List<WorkflowInstance> workflowInstances = workflowInstanceMapper.queryByWorkerGroupNameAndStatus(
+                workerGroup.getName(),
+                WorkflowExecutionStatus.getNotTerminalStatus());
         if (CollectionUtils.isNotEmpty(workflowInstances)) {
             List<Integer> workflowInstanceIds =
                     workflowInstances.stream().map(WorkflowInstance::getId).collect(Collectors.toList());

diff --git a/...main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/...main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java
@@ -118,7 +118,6 @@
 import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters;
 import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
-import org.apache.dolphinscheduler.plugin.task.sql.SqlTaskChannelFactory;
 import org.apache.dolphinscheduler.service.model.TaskNode;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
@@ -1061,9 +1060,8 @@ private void workflowDefinitionUsedInOtherTaskValid(WorkflowDefinition workflowD
         }
 
         // check workflow instances is already running
-        List<WorkflowInstance> workflowInstances = workflowInstanceService
-                .queryByWorkflowDefinitionCodeAndStatus(workflowDefinition.getCode(),
-                        org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES);
+        List<WorkflowInstance> workflowInstances = workflowInstanceService.queryByWorkflowDefinitionCodeAndStatus(
+                workflowDefinition.getCode(), WorkflowExecutionStatus.getNotTerminalStatus());
         if (CollectionUtils.isNotEmpty(workflowInstances)) {
             throw new ServiceException(Status.DELETE_WORKFLOW_DEFINITION_EXECUTING_FAIL, workflowInstances.size());
         }
@@ -1438,7 +1436,7 @@ private TaskDefinitionLog buildNormalSqlTaskDefinition(String taskName, DataSour
         sqlParameters.setLocalParams(Collections.emptyList());
         taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters));
         taskDefinition.setCode(CodeGenerateUtils.genCode());
-        taskDefinition.setTaskType(SqlTaskChannelFactory.NAME);
+        taskDefinition.setTaskType("SQL");
         taskDefinition.setFailRetryTimes(0);
         taskDefinition.setFailRetryInterval(0);
         taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE);

diff --git a/...i/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/...i/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java
@@ -24,6 +24,7 @@
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
 import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
@@ -134,7 +135,7 @@ public void testDeleteById() throws Exception {
         workerGroup.setName("测试");
         Mockito.when(workerGroupMapper.selectById(12)).thenReturn(workerGroup);
         Mockito.when(workflowInstanceMapper.queryByWorkerGroupNameAndStatus("测试",
-                org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES))
+                WorkflowExecutionStatus.getNotTerminalStatus()))
                 .thenReturn(null);
         Mockito.when(workerGroupMapper.deleteById(12)).thenReturn(1);
         Mockito.when(workflowInstanceMapper.updateWorkflowInstanceByWorkerGroupName("测试", "")).thenReturn(1);

diff --git a/...cheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/...cheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
@@ -32,6 +32,7 @@
 import org.apache.dolphinscheduler.api.utils.PageInfo;
 import org.apache.dolphinscheduler.common.enums.AuthorizationType;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
 import org.apache.dolphinscheduler.dao.entity.Queue;
 import org.apache.dolphinscheduler.dao.entity.Schedule;
 import org.apache.dolphinscheduler.dao.entity.Tenant;
@@ -191,7 +192,7 @@ public void testDeleteTenantById() {
                 baseServiceLogger)).thenReturn(true);
         when(tenantMapper.queryById(1)).thenReturn(getTenant());
         when(workflowInstanceMapper.queryByTenantCodeAndStatus(tenantCode,
-                org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES))
+                WorkflowExecutionStatus.getNotTerminalStatus()))
                         .thenReturn(getInstanceList());
         when(scheduleMapper.queryScheduleListByTenant(tenantCode)).thenReturn(getScheduleList());
         when(userMapper.queryUserListByTenant(3)).thenReturn(getUserList());

diff --git a/...ler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/...ler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
@@ -29,6 +29,7 @@
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.AuthorizationType;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
 import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
@@ -243,7 +244,7 @@ public void giveRunningProcess_whenDeleteWorkerGroupById_expectFailed() {
         List<WorkflowInstance> workflowInstances = new ArrayList<WorkflowInstance>();
         workflowInstances.add(workflowInstance);
         Mockito.when(workflowInstanceMapper.queryByWorkerGroupNameAndStatus(workerGroup.getName(),
-                org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES))
+                WorkflowExecutionStatus.getNotTerminalStatus()))
                 .thenReturn(workflowInstances);
 
         Map<String, Object> deleteFailed = workerGroupService.deleteWorkerGroupById(loginUser, 1);
@@ -261,7 +262,7 @@ public void giveValidParams_whenDeleteWorkerGroupById_expectSuccess() {
         WorkerGroup workerGroup = getWorkerGroup(1);
         Mockito.when(workerGroupMapper.selectById(1)).thenReturn(workerGroup);
         Mockito.when(workflowInstanceMapper.queryByWorkerGroupNameAndStatus(workerGroup.getName(),
-                org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(null);
+                WorkflowExecutionStatus.getNotTerminalStatus())).thenReturn(null);
 
         Mockito.when(workerGroupMapper.deleteById(1)).thenReturn(1);
 

diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml
@@ -389,8 +389,8 @@
             </dependency>
 
             <dependency>
-                <groupId>mysql</groupId>
-                <artifactId>mysql-connector-java</artifactId>
+                <groupId>com.mysql</groupId>
+                <artifactId>mysql-connector-j</artifactId>
                 <version>${mysql-connector.version}</version>
                 <scope>test</scope>
             </dependency>

diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/pom.xml b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/pom.xml
@@ -33,8 +33,8 @@
         </dependency>
 
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
     </dependencies>
 

diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml
@@ -47,8 +47,8 @@
         </dependency>
 
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
     </dependencies>