From c56784be31e7a0dcf44168e4530f275cdf0ec581 Mon Sep 17 00:00:00 2001 From: gaoyan Date: Sat, 25 Nov 2023 12:53:26 +0800 Subject: [PATCH] Add job overview (#2573) * Optimize the process * add job overview --- .../controller/JobInstanceController.java | 12 +- .../dinky/controller/MonitorController.java | 6 +- .../data/model/home/JobInstanceStatus.java | 109 +------ .../org/dinky/data/vo/task/JobInstanceVo.java | 102 +++++++ .../org/dinky/mapper/JobInstanceMapper.java | 3 +- .../org/dinky/mybatis/mapper/SuperMapper.java | 4 +- .../org/dinky/service/JobInstanceService.java | 6 +- .../service/impl/JobInstanceServiceImpl.java | 22 +- .../resources/mapper/JobInstanceMapper.xml | 15 +- .../src/components/Icons/DevopsIcons.tsx | 266 ++++++++++++++++++ dinky-web/src/locales/en-US/pages.ts | 7 + dinky-web/src/locales/zh-CN/pages.ts | 8 + .../{ => JobHistoryList}/JobHistoryList.tsx | 10 +- .../components/Overview/StatisticsCard.tsx | 52 ++++ .../JobList/components/Overview/index.tsx | 135 +++++++++ dinky-web/src/pages/DevOps/JobList/index.tsx | 99 ++++--- dinky-web/src/pages/DevOps/function.tsx | 37 +-- dinky-web/src/pages/DevOps/index.tsx | 14 +- dinky-web/src/types/Home/data.d.ts | 1 + 19 files changed, 689 insertions(+), 219 deletions(-) create mode 100644 dinky-admin/src/main/java/org/dinky/data/vo/task/JobInstanceVo.java create mode 100644 dinky-web/src/components/Icons/DevopsIcons.tsx rename dinky-web/src/pages/DevOps/JobList/components/{ => JobHistoryList}/JobHistoryList.tsx (94%) create mode 100644 dinky-web/src/pages/DevOps/JobList/components/Overview/StatisticsCard.tsx create mode 100644 dinky-web/src/pages/DevOps/JobList/components/Overview/index.tsx diff --git a/dinky-admin/src/main/java/org/dinky/controller/JobInstanceController.java b/dinky-admin/src/main/java/org/dinky/controller/JobInstanceController.java index e749d36086..f4136ba727 100644 --- a/dinky-admin/src/main/java/org/dinky/controller/JobInstanceController.java +++ b/dinky-admin/src/main/java/org/dinky/controller/JobInstanceController.java @@ -26,9 +26,10 @@ import org.dinky.data.model.ID; import org.dinky.data.model.devops.TaskManagerConfiguration; import org.dinky.data.model.ext.JobInfoDetail; -import org.dinky.data.model.job.JobInstance; +import org.dinky.data.model.home.JobInstanceStatus; import org.dinky.data.result.ProTableResult; import org.dinky.data.result.Result; +import org.dinky.data.vo.task.JobInstanceVo; import org.dinky.explainer.lineage.LineageResult; import org.dinky.service.JobInstanceService; import org.dinky.utils.BuildConfiguration; @@ -78,7 +79,7 @@ public class JobInstanceController { paramType = "body", required = true, dataTypeClass = JsonNode.class) - public ProTableResult listJobInstances(@RequestBody JsonNode para) { + public ProTableResult listJobInstances(@RequestBody JsonNode para) { return jobInstanceService.listJobInstances(para); } @@ -87,11 +88,8 @@ public ProTableResult listJobInstances(@RequestBody JsonNode para) */ @GetMapping("/getStatusCount") @ApiOperation("Get status count") - public Result getStatusCount() { - Dict result = Dict.create() - .set("history", jobInstanceService.getStatusCount(true)) - .set("instance", jobInstanceService.getStatusCount(false)); - return Result.succeed(result); + public Result getStatusCount() { + return Result.succeed(jobInstanceService.getStatusCount()); } /** diff --git a/dinky-admin/src/main/java/org/dinky/controller/MonitorController.java b/dinky-admin/src/main/java/org/dinky/controller/MonitorController.java index 83993e2217..9ecad3a02f 100644 --- a/dinky-admin/src/main/java/org/dinky/controller/MonitorController.java +++ b/dinky-admin/src/main/java/org/dinky/controller/MonitorController.java @@ -25,10 +25,10 @@ import org.dinky.data.enums.MetricsType; import org.dinky.data.metrics.Jvm; import org.dinky.data.model.Metrics; -import org.dinky.data.model.job.JobInstance; import org.dinky.data.result.ProTableResult; import org.dinky.data.result.Result; import org.dinky.data.vo.MetricsVO; +import org.dinky.data.vo.task.JobInstanceVo; import org.dinky.service.JobInstanceService; import org.dinky.service.MonitorService; @@ -93,9 +93,9 @@ public Result> getFlinkData(@RequestParam Long startTime, Long e ObjectNode para = nodeFactory.objectNode(); para.put("isHistory", false); para.put("taskId", taskIds); - ProTableResult jobInstanceProTableResult = jobInstanceService.listJobInstances(para); + ProTableResult jobInstanceProTableResult = jobInstanceService.listJobInstances(para); List jids = jobInstanceProTableResult.getData().stream() - .map(JobInstance::getJid) + .map(JobInstanceVo::getJid) .collect(Collectors.toList()); return Result.succeed(monitorService.getData( DateUtil.date(startTime), diff --git a/dinky-admin/src/main/java/org/dinky/data/model/home/JobInstanceStatus.java b/dinky-admin/src/main/java/org/dinky/data/model/home/JobInstanceStatus.java index 4ae642f8e2..2afc4cbba6 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/home/JobInstanceStatus.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/home/JobInstanceStatus.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.Data; /** * JobInstanceStatus @@ -28,6 +29,7 @@ * @since 2022/2/28 22:25 */ @ApiModel(value = "JobInstanceStatus", description = "Job Instance Status Information") +@Data public class JobInstanceStatus { @ApiModelProperty(value = "All", dataType = "Integer", example = "10", notes = "Total count of job instances") @@ -117,109 +119,6 @@ public class JobInstanceStatus { notes = "Count of job instances in the Unknown state") private Integer unknown = 0; - public JobInstanceStatus() {} - - public Integer getAll() { - return all; - } - - public void setAll(Integer all) { - this.all = all; - } - - public Integer getInitializing() { - return initializing; - } - - public void setInitializing(Integer initializing) { - this.initializing = initializing; - } - - public Integer getRunning() { - return running; - } - - public void setRunning(Integer running) { - this.running = running; - } - - public Integer getFinished() { - return finished; - } - - public void setFinished(Integer finished) { - this.finished = finished; - } - - public Integer getFailed() { - return failed; - } - - public void setFailed(Integer failed) { - this.failed = failed; - } - - public Integer getCanceled() { - return canceled; - } - - public void setCanceled(Integer canceled) { - this.canceled = canceled; - } - - public Integer getRestarting() { - return restarting; - } - - public void setRestarting(Integer restarting) { - this.restarting = restarting; - } - - public Integer getCreated() { - return created; - } - - public void setCreated(Integer created) { - this.created = created; - } - - public Integer getFailing() { - return failing; - } - - public void setFailing(Integer failing) { - this.failing = failing; - } - - public Integer getCancelling() { - return cancelling; - } - - public void setCancelling(Integer cancelling) { - this.cancelling = cancelling; - } - - public Integer getSuspended() { - return suspended; - } - - public void setSuspended(Integer suspended) { - this.suspended = suspended; - } - - public Integer getReconciling() { - return reconciling; - } - - public void setReconciling(Integer reconciling) { - this.reconciling = reconciling; - } - - public Integer getUnknown() { - return unknown; - } - - public void setUnknown(Integer unknown) { - this.unknown = unknown; - } + @ApiModelProperty(value = "modelOverview", dataType = "JobModelOverview", notes = "batch and steamng count") + private JobModelOverview modelOverview; } diff --git a/dinky-admin/src/main/java/org/dinky/data/vo/task/JobInstanceVo.java b/dinky-admin/src/main/java/org/dinky/data/vo/task/JobInstanceVo.java new file mode 100644 index 0000000000..80635318a5 --- /dev/null +++ b/dinky-admin/src/main/java/org/dinky/data/vo/task/JobInstanceVo.java @@ -0,0 +1,102 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dinky.data.vo.task; + +import java.time.LocalDateTime; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.fasterxml.jackson.annotation.JsonFormat; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@Data +public class JobInstanceVo { + + @TableId(value = "id", type = IdType.AUTO) + @ApiModelProperty( + value = "ID", + dataType = "Integer", + example = "1", + notes = "Unique identifier for the job instance") + private Integer id; + + @ApiModelProperty(value = "Name", dataType = "String", notes = "Name of the job instance") + private String name; + + @ApiModelProperty( + value = "Task ID", + dataType = "Integer", + example = "1", + notes = "Task ID associated with the job instance") + private Integer taskId; + + @ApiModelProperty(value = "Step", dataType = "Integer", example = "1", notes = "Step number of the job instance") + private Integer step; + + @ApiModelProperty(value = "JID", dataType = "String", notes = "JID of the job instance") + private String jid; + + @ApiModelProperty(value = "Status", dataType = "String", notes = "Status of the job instance") + private String status; + + @ApiModelProperty(value = "type", dataType = "String", notes = "run mode type", example = "kubernets-application") + private String type; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @TableField(fill = FieldFill.INSERT) + @ApiModelProperty( + value = "Create Time", + dataType = "String", + notes = "Timestamp indicating the creation time of the job instance") + private LocalDateTime createTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @TableField(fill = FieldFill.INSERT_UPDATE) + @ApiModelProperty( + value = "Update Time", + dataType = "String", + notes = "Timestamp indicating the last update time of the job instance") + private LocalDateTime updateTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @ApiModelProperty( + value = "Finish Time", + dataType = "String", + notes = "Timestamp indicating the finish time of the job instance") + private LocalDateTime finishTime; + + @ApiModelProperty( + value = "Duration", + dataType = "Long", + example = "3600", + notes = "Duration of the job instance in seconds") + private Long duration; + + @ApiModelProperty( + value = "Failed Restart Count", + dataType = "Integer", + example = "2", + notes = "Count of failed restarts") + private Integer failedRestartCount; +} diff --git a/dinky-admin/src/main/java/org/dinky/mapper/JobInstanceMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/JobInstanceMapper.java index df0546a899..55fd072a3e 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/JobInstanceMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/JobInstanceMapper.java @@ -20,6 +20,7 @@ package org.dinky.mapper; import org.dinky.data.model.home.JobInstanceCount; +import org.dinky.data.model.home.JobModelOverview; import org.dinky.data.model.job.JobInstance; import org.dinky.mybatis.mapper.SuperMapper; @@ -43,7 +44,7 @@ public interface JobInstanceMapper extends SuperMapper { List countStatus(); - List countHistoryStatus(); + JobModelOverview getJobStreamingOrBatchModelOverview(); @InterceptorIgnore(tenantLine = "true") List listJobInstanceActive(); diff --git a/dinky-admin/src/main/java/org/dinky/mybatis/mapper/SuperMapper.java b/dinky-admin/src/main/java/org/dinky/mybatis/mapper/SuperMapper.java index 07944e3731..756f39b0e5 100644 --- a/dinky-admin/src/main/java/org/dinky/mybatis/mapper/SuperMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mybatis/mapper/SuperMapper.java @@ -36,6 +36,6 @@ */ public interface SuperMapper extends BaseMapper { - List selectForProTable( - Page page, @Param(Constants.WRAPPER) Wrapper queryWrapper, @Param("param") Map param); + List selectForProTable( + Page page, @Param(Constants.WRAPPER) Wrapper queryWrapper, @Param("param") Map param); } diff --git a/dinky-admin/src/main/java/org/dinky/service/JobInstanceService.java b/dinky-admin/src/main/java/org/dinky/service/JobInstanceService.java index 1ce6dd52b5..204ccafc55 100644 --- a/dinky-admin/src/main/java/org/dinky/service/JobInstanceService.java +++ b/dinky-admin/src/main/java/org/dinky/service/JobInstanceService.java @@ -23,6 +23,7 @@ import org.dinky.data.model.home.JobInstanceStatus; import org.dinky.data.model.job.JobInstance; import org.dinky.data.result.ProTableResult; +import org.dinky.data.vo.task.JobInstanceVo; import org.dinky.explainer.lineage.LineageResult; import org.dinky.mybatis.service.ISuperService; @@ -49,10 +50,9 @@ public interface JobInstanceService extends ISuperService { /** * Get the count of job instance statuses. * - * @param isHistory A boolean indicating whether to get the history status count or not. * @return An integer representing the count of job instance statuses. */ - JobInstanceStatus getStatusCount(boolean isHistory); + JobInstanceStatus getStatusCount(); /** * Get a list of active job instances. @@ -123,7 +123,7 @@ public interface JobInstanceService extends ISuperService { * @param para A {@link JsonNode} object representing the filter conditions for listing job instances. * @return A {@link ProTableResult} object representing the list of job instances. */ - ProTableResult listJobInstances(JsonNode para); + ProTableResult listJobInstances(JsonNode para); /** * Initialize the tenant by the given job instance ID. diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/JobInstanceServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/JobInstanceServiceImpl.java index 19c17416b7..8ae38f837d 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/JobInstanceServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/JobInstanceServiceImpl.java @@ -33,9 +33,11 @@ import org.dinky.data.model.ext.JobInfoDetail; import org.dinky.data.model.home.JobInstanceCount; import org.dinky.data.model.home.JobInstanceStatus; +import org.dinky.data.model.home.JobModelOverview; import org.dinky.data.model.job.History; import org.dinky.data.model.job.JobInstance; import org.dinky.data.result.ProTableResult; +import org.dinky.data.vo.task.JobInstanceVo; import org.dinky.explainer.lineage.LineageBuilder; import org.dinky.explainer.lineage.LineageResult; import org.dinky.job.FlinkJobTask; @@ -89,14 +91,12 @@ public JobInstance getByIdWithoutTenant(Integer id) { } @Override - public JobInstanceStatus getStatusCount(boolean isHistory) { + public JobInstanceStatus getStatusCount() { List jobInstanceCounts; - if (isHistory) { - jobInstanceCounts = baseMapper.countHistoryStatus(); - } else { - jobInstanceCounts = baseMapper.countStatus(); - } + jobInstanceCounts = baseMapper.countStatus(); + JobModelOverview modelOverview = baseMapper.getJobStreamingOrBatchModelOverview(); JobInstanceStatus jobInstanceStatus = new JobInstanceStatus(); + jobInstanceStatus.setModelOverview(modelOverview); int total = 0; for (JobInstanceCount item : jobInstanceCounts) { Integer counts = Asserts.isNull(item.getCounts()) ? 0 : item.getCounts(); @@ -252,16 +252,16 @@ public JobInstance getJobInstanceByTaskId(Integer id) { } @Override - public ProTableResult listJobInstances(JsonNode para) { + public ProTableResult listJobInstances(JsonNode para) { int current = para.has("current") ? para.get("current").asInt() : 1; int pageSize = para.has("pageSize") ? para.get("pageSize").asInt() : 10; - QueryWrapper queryWrapper = new QueryWrapper<>(); + QueryWrapper queryWrapper = new QueryWrapper<>(); ProTableUtil.autoQueryDefalut(para, queryWrapper); ObjectMapper mapper = new ObjectMapper(); Map param = mapper.convertValue(para, Map.class); - Page page = new Page<>(current, pageSize); - List list = baseMapper.selectForProTable(page, queryWrapper, param); - return ProTableResult.builder() + Page page = new Page<>(current, pageSize); + List list = baseMapper.selectForProTable(page, queryWrapper, param); + return ProTableResult.builder() .success(true) .data(list) .total(page.getTotal()) diff --git a/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml b/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml index 655aa92411..b008c171b6 100644 --- a/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml +++ b/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml @@ -9,7 +9,7 @@ limit 1 - select a.*, dh.type as type, @@ -72,11 +72,14 @@ group by status - + select sum(IF(b.batch, 1, 0)) as batchJobCount, + sum(IF(b.batch, 0, 1)) as streamingJobCount + from (select json_extract(dh.config_json, '$.batchModel') as batch + from (select max(ji.id) as id from dinky_job_instance ji group by ji.task_id) as a + left join dinky_job_instance dji on dji.id = a.id + left join dinky.dinky_history dh on dji.history_id = dh.id) b + group by b.batch