diff --git a/dinky-admin/src/main/java/org/dinky/data/model/Dashboard.java b/dinky-admin/src/main/java/org/dinky/data/model/Dashboard.java index afe120f0b3..5e387c7e96 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/Dashboard.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/Dashboard.java @@ -25,6 +25,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.extension.activerecord.Model; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -59,6 +60,10 @@ public class Dashboard extends Model { @ApiModelProperty(value = "Layouts", dataType = "String", example = "[]", notes = "Layouts of the metrics") private String layouts; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @TableField(fill = FieldFill.INSERT) @JsonDeserialize(using = LocalDateTimeDeserializer.class) @JsonSerialize(using = LocalDateTimeSerializer.class) diff --git a/dinky-admin/src/main/java/org/dinky/data/model/LoginLog.java b/dinky-admin/src/main/java/org/dinky/data/model/LoginLog.java index 8876bc75d4..2764ad45f6 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/LoginLog.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/LoginLog.java @@ -34,7 +34,9 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data @ApiModel(value = "LoginLog", description = "Login Log Information") @TableName("dinky_sys_login_log") @@ -85,9 +87,6 @@ public class LoginLog extends DateBaseEntity implements Serializable { @ApiModelProperty(value = "Message", dataType = "String", notes = "Additional message or details about the login") private String msg; - @ApiModelProperty(value = "Is Deleted", dataType = "Boolean", notes = "Flag indicating if the login log is deleted") - private Boolean isDeleted; - @ApiModelProperty(value = "Access Time", dataType = "String", notes = "Timestamp indicating the time of login") private LocalDateTime accessTime; } diff --git a/dinky-admin/src/main/java/org/dinky/data/model/Metrics.java b/dinky-admin/src/main/java/org/dinky/data/model/Metrics.java index e2def9d4dc..4c6335567d 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/Metrics.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/Metrics.java @@ -25,6 +25,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.extension.activerecord.Model; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -93,6 +94,10 @@ public class Metrics extends Model { notes = "Timestamp indicating the last update time of the metrics") private LocalDateTime updateTime; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @TableField(exist = false) private static final long serialVersionUID = 1L; diff --git a/dinky-admin/src/main/java/org/dinky/data/model/OperateLog.java b/dinky-admin/src/main/java/org/dinky/data/model/OperateLog.java index 43612b5340..9a5ee304f4 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/OperateLog.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/OperateLog.java @@ -24,6 +24,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; @@ -49,6 +50,10 @@ public class OperateLog implements Serializable { notes = "Name of the operation module") private String moduleName; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @ApiModelProperty( value = "Business Type", dataType = "Integer", diff --git a/dinky-admin/src/main/java/org/dinky/data/model/Resources.java b/dinky-admin/src/main/java/org/dinky/data/model/Resources.java index ecbc133b4a..84b6a5bd40 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/Resources.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/Resources.java @@ -27,6 +27,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.extension.activerecord.Model; import com.fasterxml.jackson.annotation.JsonFormat; @@ -105,6 +106,10 @@ public class Resources extends Model { notes = "Flag indicating if the resource is a directory") private Boolean isDirectory; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @TableField(fill = FieldFill.INSERT) @JsonDeserialize(using = LocalDateTimeDeserializer.class) @JsonSerialize(using = LocalDateTimeSerializer.class) diff --git a/dinky-admin/src/main/java/org/dinky/data/model/Savepoints.java b/dinky-admin/src/main/java/org/dinky/data/model/Savepoints.java index 10e010a721..94bdc7d6e8 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/Savepoints.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/Savepoints.java @@ -30,6 +30,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -95,6 +96,10 @@ public class Savepoints implements Serializable { notes = "Timestamp indicating the creation time of the savepoint") private LocalDateTime createTime; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @TableField(fill = FieldFill.INSERT) @ApiModelProperty(value = "Creator", dataType = "String", notes = "Creator of the savepoint") private Integer creator; diff --git a/dinky-admin/src/main/java/org/dinky/data/model/SysToken.java b/dinky-admin/src/main/java/org/dinky/data/model/SysToken.java index 122d98ba04..e9e7b37bfd 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/SysToken.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/SysToken.java @@ -36,9 +36,11 @@ import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; +@EqualsAndHashCode(callSuper = true) @Data @AllArgsConstructor @NoArgsConstructor diff --git a/dinky-admin/src/main/java/org/dinky/data/model/TaskVersion.java b/dinky-admin/src/main/java/org/dinky/data/model/TaskVersion.java index 442a4dc5ca..90b3f13d4e 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/TaskVersion.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/TaskVersion.java @@ -27,6 +27,7 @@ import com.baomidou.mybatisplus.annotation.FieldFill; import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -60,6 +61,10 @@ public class TaskVersion implements Serializable { @TableField(value = "task_id") private Integer taskId; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @ApiModelProperty(value = "Version ID", dataType = "Integer", notes = "ID of the version") @TableField(value = "version_id") private Integer versionId; diff --git a/dinky-admin/src/main/java/org/dinky/data/model/alert/AlertHistory.java b/dinky-admin/src/main/java/org/dinky/data/model/alert/AlertHistory.java index dcd0da6cc5..3812a7d177 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/alert/AlertHistory.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/alert/AlertHistory.java @@ -26,6 +26,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -85,6 +86,10 @@ public class AlertHistory implements Serializable { @JsonSerialize(using = LocalDateTimeSerializer.class) private LocalDateTime createTime; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @TableField(fill = FieldFill.INSERT_UPDATE) @ApiModelProperty(value = "Update Time", example = "2022-02-24 20:12:00", dataType = "LocalDateTime") @JsonDeserialize(using = LocalDateTimeDeserializer.class) diff --git a/dinky-admin/src/main/java/org/dinky/data/model/job/History.java b/dinky-admin/src/main/java/org/dinky/data/model/job/History.java index 542544e6da..08bde90f87 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/job/History.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/job/History.java @@ -27,6 +27,7 @@ import com.baomidou.mybatisplus.annotation.FieldStrategy; import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -129,6 +130,10 @@ public class History implements Serializable { @ApiModelProperty(value = "Group by count", dataType = "Integer") private Long count; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @ApiModelProperty(hidden = true) public JobInstance buildJobInstance() { JobInstance jobInstance = new JobInstance(); diff --git a/dinky-admin/src/main/java/org/dinky/data/model/job/JobHistory.java b/dinky-admin/src/main/java/org/dinky/data/model/job/JobHistory.java index 6e10950f35..302d830d79 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/job/JobHistory.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/job/JobHistory.java @@ -33,6 +33,7 @@ import com.baomidou.mybatisplus.annotation.FieldFill; import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -81,6 +82,10 @@ public class JobHistory implements Serializable { @TableField(typeHandler = JSONObjectHandler.class) private FlinkJobDetailInfo jobJson; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @ApiModelProperty( value = "Exceptions JSON", dataType = "String", diff --git a/dinky-admin/src/main/java/org/dinky/data/model/job/JobInstance.java b/dinky-admin/src/main/java/org/dinky/data/model/job/JobInstance.java index 60a056708e..22aaeafd9c 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/job/JobInstance.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/job/JobInstance.java @@ -27,6 +27,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -103,6 +104,10 @@ public class JobInstance implements Serializable { @ApiModelProperty(value = "Error", dataType = "String", notes = "Error message associated with the job instance") private String error; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") @TableField(fill = FieldFill.INSERT) @JsonDeserialize(using = LocalDateTimeDeserializer.class) diff --git a/dinky-admin/src/main/java/org/dinky/data/model/rbac/Role.java b/dinky-admin/src/main/java/org/dinky/data/model/rbac/Role.java index 225bbac902..fbd1b2b00c 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/rbac/Role.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/rbac/Role.java @@ -63,12 +63,6 @@ public class Role extends DateBaseEntity implements Serializable { @ApiModelProperty(value = "Role Name", dataType = "String", example = "Administrator", notes = "Name of the role") private String roleName; - @ApiModelProperty( - value = "Is Delete", - dataType = "Boolean", - notes = "Flag indicating if the role is marked as deleted") - private Boolean isDelete; - @ApiModelProperty(value = "Note", dataType = "String", notes = "Additional notes or details about the role") private String note; diff --git a/dinky-admin/src/main/java/org/dinky/data/model/rbac/Tenant.java b/dinky-admin/src/main/java/org/dinky/data/model/rbac/Tenant.java index b2e453aabb..b81ba246a4 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/rbac/Tenant.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/rbac/Tenant.java @@ -52,8 +52,4 @@ public class Tenant extends DateBaseEntity implements Serializable { /** note */ @ApiModelProperty(value = "Tenant Note", required = true, dataType = "String", example = "Default") private String note; - - /** is delete */ - @ApiModelProperty(value = "Is Delete", required = true, dataType = "Boolean", example = "false") - private Boolean isDelete; } diff --git a/dinky-admin/src/main/java/org/dinky/data/model/rbac/User.java b/dinky-admin/src/main/java/org/dinky/data/model/rbac/User.java index 1c93871f7b..16221fcd61 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/rbac/User.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/rbac/User.java @@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.annotation.TableName; import io.swagger.annotations.ApiModel; @@ -80,10 +79,6 @@ public class User extends DateBaseEntity implements Serializable { @ApiModelProperty(value = "Enabled", dataType = "Boolean", notes = "Whether the user is enabled") private Boolean enabled; - @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") - @TableLogic - private Boolean isDelete; - @ApiModelProperty(value = "Super Admin Flag", dataType = "Boolean", notes = "Whether the user is a super admin") private Boolean superAdminFlag; diff --git a/dinky-admin/src/main/java/org/dinky/interceptor/PostgreSQLPrepareInterceptor.java b/dinky-admin/src/main/java/org/dinky/interceptor/PostgreSQLPrepareInterceptor.java index 1ac29a1362..4a8bb4c8ba 100644 --- a/dinky-admin/src/main/java/org/dinky/interceptor/PostgreSQLPrepareInterceptor.java +++ b/dinky-admin/src/main/java/org/dinky/interceptor/PostgreSQLPrepareInterceptor.java @@ -31,12 +31,15 @@ import java.sql.Connection; import java.util.Properties; +import lombok.extern.slf4j.Slf4j; + @Intercepts({ @Signature( type = StatementHandler.class, method = "prepare", args = {Connection.class, Integer.class}) }) +@Slf4j public class PostgreSQLPrepareInterceptor implements Interceptor { @Override public Object intercept(final Invocation invocation) throws Throwable { @@ -45,6 +48,10 @@ public Object intercept(final Invocation invocation) throws Throwable { Field field = boundSql.getClass().getDeclaredField("sql"); field.setAccessible(true); field.set(boundSql, boundSql.getSql().replace("`", "\"").toLowerCase()); + field.set(boundSql, boundSql.getSql().replaceAll("is_delete = 0", "is_delete = false")); + field.set(boundSql, boundSql.getSql().replaceAll("is_delete = 1", "is_delete = true")); + field.set(boundSql, boundSql.getSql().replaceAll("set is_delete = 0", "set is_delete = false")); + field.set(boundSql, boundSql.getSql().replaceAll("set is_delete = 1", "set is_delete = true")); return invocation.proceed(); } diff --git a/dinky-admin/src/main/java/org/dinky/mapper/MenuMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/MenuMapper.java index e7575718c1..95e1a56814 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/MenuMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/MenuMapper.java @@ -33,10 +33,6 @@ public interface MenuMapper extends SuperMapper { List listAllMenus(); - List selectMenuList(Menu menu); - - List selectMenuPerms(); - List selectMenuListByUserId(Menu menu); List selectMenuPermsByUserId(Integer userId); @@ -45,11 +41,7 @@ public interface MenuMapper extends SuperMapper { List selectMenuTreeByUserId(@Param("userId") Integer userId); - List selectMenuListByRoleId(Integer roleId); - int hasChildByMenuId(Integer menuId); - Menu checkMenuNameUnique(@Param("name") String name, @Param("parentId") Integer parentId); - List selectMenuPermsByRoleId(Integer roleId); } diff --git a/dinky-admin/src/main/java/org/dinky/mapper/RoleMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/RoleMapper.java index d3422cdc51..da274f62a8 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/RoleMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/RoleMapper.java @@ -23,19 +23,13 @@ import org.dinky.mybatis.mapper.SuperMapper; import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; import java.util.List; -import java.util.Set; /** role mapper interface */ @Mapper public interface RoleMapper extends SuperMapper { - List getRoleByIds(@Param("roleIds") Set roleIds); - - List getRoleByTenantIdAndIds(@Param("tenantId") String tenantId, @Param("roleIds") Set roleIds); - /** * Query roles by user ID. * diff --git a/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java index c346ef9990..8e91c0ec59 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java @@ -24,15 +24,9 @@ import org.apache.ibatis.annotations.Mapper; -import java.util.List; - /** MenuMapper */ @Mapper public interface RoleMenuMapper extends SuperMapper { int checkMenuExistRole(Integer menuId); - - int deleteRoleMenuByRoleId(Integer roleId); - - int batchRoleMenu(List roleMenuList); } diff --git a/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java index 60b57c77ea..3a171703a3 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java @@ -39,12 +39,6 @@ @Mapper public interface TaskMapper extends SuperMapper { - List queryOnLineTaskByDoneStatus( - @Param("parentIds") List parentIds, - @Param("stepIds") List stepIds, - @Param("includeNull") boolean includeNull, - @Param("jobStatuses") List jobStatuses); - @InterceptorIgnore(tenantLine = "true") Task getTaskByNameAndTenantId(@Param("name") String name, @Param("tenantId") Integer tenantId); diff --git a/dinky-admin/src/main/java/org/dinky/mapper/TenantMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/TenantMapper.java index 2c563a2758..04b8234370 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/TenantMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/TenantMapper.java @@ -23,14 +23,7 @@ import org.dinky.mybatis.mapper.SuperMapper; import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; -import java.util.Set; /** tenant mapper interface */ @Mapper -public interface TenantMapper extends SuperMapper { - - List getTenantByIds(@Param("tenantIds") Set tenantIds); -} +public interface TenantMapper extends SuperMapper {} diff --git a/dinky-admin/src/main/java/org/dinky/mapper/UserMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/UserMapper.java index c9be17bc93..31ee9bff43 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/UserMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/UserMapper.java @@ -33,7 +33,5 @@ @Mapper public interface UserMapper extends SuperMapper { - Integer queryAdminUserByTenant(@Param("tenantId") Integer tenantId); - Integer recoveryUser(@Param("id") Integer userId); } diff --git a/dinky-admin/src/main/java/org/dinky/mybatis/model/DateBaseEntity.java b/dinky-admin/src/main/java/org/dinky/mybatis/model/DateBaseEntity.java index 15df518f81..7eb742da2f 100644 --- a/dinky-admin/src/main/java/org/dinky/mybatis/model/DateBaseEntity.java +++ b/dinky-admin/src/main/java/org/dinky/mybatis/model/DateBaseEntity.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.FieldFill; import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.extension.activerecord.Model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -56,6 +57,10 @@ public class DateBaseEntity> extends Model { example = "2021-05-28 00:00:00") private LocalDateTime createTime; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @TableField(fill = FieldFill.INSERT_UPDATE) @JsonDeserialize(using = LocalDateTimeDeserializer.class) @JsonSerialize(using = LocalDateTimeSerializer.class) diff --git a/dinky-admin/src/main/java/org/dinky/mybatis/model/SuperEntity.java b/dinky-admin/src/main/java/org/dinky/mybatis/model/SuperEntity.java index 8ec726cf04..c42c2a1ed7 100644 --- a/dinky-admin/src/main/java/org/dinky/mybatis/model/SuperEntity.java +++ b/dinky-admin/src/main/java/org/dinky/mybatis/model/SuperEntity.java @@ -30,6 +30,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; import com.baomidou.mybatisplus.extension.activerecord.Model; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -51,7 +52,9 @@ @ApiModel(value = "SuperEntity", description = "Super Base Entity", parent = Model.class) public class SuperEntity> extends Model { - /** 主键ID */ + /** + * 主键ID + */ @TableId(value = "id", type = IdType.AUTO) @ApiModelProperty(value = "ID", required = true, dataType = "Integer", example = "1", notes = "Primary Key") private Integer id; @@ -96,6 +99,10 @@ public class SuperEntity> extends Model { @ApiModelProperty(value = "Updater", required = true, dataType = "String", example = "Updater") private Integer updater; + @ApiModelProperty(value = "Is Delete", dataType = "Boolean", notes = "Whether the user is deleted") + @TableLogic + private Boolean isDelete; + @Override public Serializable pkVal() { return this.id; diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/DocumentServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/DocumentServiceImpl.java index b7e40c179f..f542f88472 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/DocumentServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/DocumentServiceImpl.java @@ -38,10 +38,16 @@ public class DocumentServiceImpl extends SuperServiceImpl getFillAllByVersion(String version) { if (Asserts.isNotNullString(version)) { - return baseMapper.selectList( - new QueryWrapper().eq("version", version).eq("enabled", 1)); + return baseMapper.selectList(new QueryWrapper() + .lambda() + .eq(Document::getVersion, version) + .eq(Document::getEnabled, true) + .eq(Document::getIsDelete, false)); } else { - return baseMapper.selectList(new QueryWrapper().eq("enabled", 1)); + return baseMapper.selectList(new QueryWrapper() + .lambda() + .eq(Document::getEnabled, true) + .eq(Document::getIsDelete, false)); } } diff --git a/dinky-admin/src/main/java/org/dinky/utils/SqliteUtil.java b/dinky-admin/src/main/java/org/dinky/utils/SqliteUtil.java index 328aeb71d1..87e863ac67 100644 --- a/dinky-admin/src/main/java/org/dinky/utils/SqliteUtil.java +++ b/dinky-admin/src/main/java/org/dinky/utils/SqliteUtil.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -35,12 +36,12 @@ public enum SqliteUtil { INSTANCE; private Connection connection; - private final AtomicLong lastRecyle = new AtomicLong(0); + private final AtomicLong lastRecycle = new AtomicLong(0); static { try { SqliteUtil.INSTANCE.connect("dinky.db"); - SqliteUtil.INSTANCE.recyleData(); + SqliteUtil.INSTANCE.recycleData(); } catch (SQLException e) { throw new RuntimeException(e); } @@ -56,28 +57,28 @@ public void createTable(String tableName, String columns) { try (Statement stmt = connection.createStatement()) { stmt.execute(sql); } catch (SQLException e) { - log.error("Failed to create table: " + e.getMessage()); + log.error("Failed to create table: {}", e.getMessage()); } } public void executeSql(String sql) throws SQLException { - Statement pstmt = connection.createStatement(); - pstmt.executeUpdate(sql); + Statement stmt = connection.createStatement(); + stmt.executeUpdate(sql); connection.commit(); } - public void recyleData() { + public void recycleData() { long now = System.currentTimeMillis(); - if (now - lastRecyle.get() < 1000 * 60 * 60) { + if (now - lastRecycle.get() < 1000 * 60 * 60) { return; } - lastRecyle.set(now); + lastRecycle.set(now); try { String sql = "DELETE FROM dinky_metrics WHERE heart_time <= datetime('now', '-7 days')"; executeSql(sql); executeSql("VACUUM"); } catch (SQLException e) { - log.error("Failed to recyle database: " + e.getMessage()); + log.error("Failed to recycle database: {}", e.getMessage()); } } @@ -95,9 +96,9 @@ public void write(String tableName, List columns, List> val pstmt.executeBatch(); connection.commit(); } catch (SQLException e) { - log.error("Failed to write to SQLite: " + e.getMessage()); + log.error("Failed to write to SQLite: {}", e.getMessage()); } - recyleData(); + recycleData(); } private static String createInsertSql(String tableName, List columns) { @@ -130,6 +131,7 @@ public void close() throws SQLException { } } + @Getter public static class PreparedResultSet implements AutoCloseable { private final PreparedStatement pstmt; private final ResultSet rs; @@ -139,14 +141,6 @@ public PreparedResultSet(PreparedStatement pstmt, ResultSet rs) { this.rs = rs; } - public PreparedStatement getPstmt() { - return pstmt; - } - - public ResultSet getRs() { - return rs; - } - @Override public void close() throws Exception { pstmt.close(); diff --git a/dinky-admin/src/main/resources/db/migration/h2/V20241230.1.3.0__release.sql b/dinky-admin/src/main/resources/db/migration/h2/V20241230.1.3.0__release.sql new file mode 100644 index 0000000000..69ca7ed76b --- /dev/null +++ b/dinky-admin/src/main/resources/db/migration/h2/V20241230.1.3.0__release.sql @@ -0,0 +1,125 @@ +ALTER TABLE `dinky_alert_history` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_history` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_job_history` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_job_instance` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_sys_menu` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_sys_role_menu` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_row_permissions` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_user_role` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_user_tenant` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_udf_manage` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_udf_template` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_catalogue` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_cluster_configuration` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_cluster` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_dashboard` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_database` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + + +ALTER TABLE `dinky_flink_document` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_fragment` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_git_project` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_metrics` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_sys_operate_log` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_resources` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_savepoints` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_sys_token` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_task` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_task_version` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_alert_group` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_alert_instance` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_alert_rules` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_alert_template` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `dinky_sys_login_log` + CHANGE COLUMN `is_deleted` `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `metadata_column` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `metadata_database` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `metadata_database_property` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `metadata_function` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `metadata_table` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE `metadata_table_property` + ADD COLUMN `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +ALTER TABLE metadata_column + ADD UNIQUE INDEX unx_metadata_column (table_id, column_name, is_delete) USING BTREE; +ALTER TABLE metadata_database + ADD UNIQUE INDEX unx_metadata_database (database_name, is_delete) USING BTREE; +ALTER TABLE metadata_database_property + ADD UNIQUE INDEX unx_metadata_database_property (database_id, `key`, is_delete) USING BTREE; +ALTER TABLE metadata_function + ADD UNIQUE INDEX unx_metadata_function (`function_name`, `database_id`, `is_delete`) USING BTREE; +ALTER TABLE metadata_table + ADD UNIQUE INDEX unx_metadata_table (`table_name`, `database_id`, `is_delete`) USING BTREE; +ALTER TABLE metadata_table_property + ADD UNIQUE INDEX unx_metadata_table_property (table_id, `key`, is_delete) USING BTREE; + diff --git a/dinky-admin/src/main/resources/db/migration/mysql/V20241230.1.3.0__release.sql b/dinky-admin/src/main/resources/db/migration/mysql/V20241230.1.3.0__release.sql new file mode 100644 index 0000000000..c38db327e8 --- /dev/null +++ b/dinky-admin/src/main/resources/db/migration/mysql/V20241230.1.3.0__release.sql @@ -0,0 +1,82 @@ +SET FOREIGN_KEY_CHECKS = 0; + +CALL add_column_if_not_exists('dinky_alert_history', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_history', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_job_history', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_job_instance', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_sys_menu', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_sys_role_menu', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_row_permissions', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_user_role', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_user_tenant', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_udf_manage', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_udf_template', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_catalogue', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_cluster_configuration', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_cluster', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_dashboard', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_database', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_flink_document', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_fragment', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_git_project', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_metrics', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_sys_operate_log', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_resources', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_savepoints', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_sys_token', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_task', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_task_version', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_alert_group', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_alert_instance', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_alert_rules', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +CALL add_column_if_not_exists('dinky_alert_template', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +ALTER TABLE `dinky_sys_login_log` + CHANGE COLUMN `is_deleted` `is_delete` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'is delete 0: false, 1: true'; + +CALL add_column_if_not_exists('metadata_column', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); +CALL add_column_if_not_exists('metadata_database', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); +CALL add_column_if_not_exists('metadata_database_property', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); +CALL add_column_if_not_exists('metadata_function', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); +CALL add_column_if_not_exists('metadata_table', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); +CALL add_column_if_not_exists('metadata_table_property', 'is_delete', 'tinyint(1)', '0', 'is delete 0: false, 1: true', null); + +ALTER TABLE metadata_column ADD UNIQUE INDEX unx_metadata_column(table_id, column_name, is_delete) USING BTREE; +ALTER TABLE metadata_database ADD UNIQUE INDEX unx_metadata_database(database_name, is_delete) USING BTREE; +ALTER TABLE metadata_database_property ADD UNIQUE INDEX unx_metadata_database_property(database_id, `key`, is_delete) USING BTREE; +ALTER TABLE metadata_function ADD UNIQUE INDEX unx_metadata_function(`function_name`, `database_id`, `is_delete`) USING BTREE; +ALTER TABLE metadata_table ADD UNIQUE INDEX unx_metadata_table(`table_name`, `database_id`, `is_delete`) USING BTREE; +ALTER TABLE metadata_table_property ADD UNIQUE INDEX unx_metadata_table_property(table_id, `key`, is_delete) USING BTREE; + + + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/dinky-admin/src/main/resources/db/migration/postgresql/V20241230.1.3.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20241230.1.3.0__release.sql new file mode 100644 index 0000000000..3b289d6c16 --- /dev/null +++ b/dinky-admin/src/main/resources/db/migration/postgresql/V20241230.1.3.0__release.sql @@ -0,0 +1,98 @@ + +SELECT add_column_if_not_exists('public','dinky_alert_history', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_history', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_job_instance', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_job_history', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_job_instance', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_sys_menu', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_sys_role_menu', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_row_permissions', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_user_role', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_user_tenant', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_udf_manage', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_udf_template', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_catalogue', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','dinky_cluster_configuration', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_cluster', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_dashboard', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_database', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_flink_document', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_fragment', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_git_project', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_metrics', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_sys_operate_log', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_resources', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_savepoints', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_sys_token', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_task', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_task_version', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_alert_group', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_alert_instance', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_alert_rules', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +select add_column_if_not_exists('public','dinky_alert_template', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +ALTER TABLE public.dinky_sys_login_log drop column is_deleted; +select add_column_if_not_exists('public','dinky_sys_login_log', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + + +SELECT add_column_if_not_exists('public','metadata_column', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','metadata_database', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','metadata_database_property', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','metadata_function', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','metadata_table', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + +SELECT add_column_if_not_exists('public','metadata_table_property', 'is_delete', 'boolean', 'false', 'is delete 0: false, 1: true'); + + +CREATE UNIQUE INDEX unx1_metadata_column + ON metadata_column USING BTREE (table_id, column_name, is_delete); + +CREATE UNIQUE INDEX unx1_metadata_database + ON metadata_database USING BTREE (database_name, is_delete); + +CREATE UNIQUE INDEX unx1_metadata_database_property + ON metadata_database_property USING BTREE (database_id, "key", is_delete); + +CREATE UNIQUE INDEX unx1_metadata_function + ON metadata_function USING BTREE ("function_name", database_id, is_delete); + +CREATE UNIQUE INDEX unx1_metadata_table + ON metadata_table USING BTREE ("table_name", database_id, is_delete); + +CREATE UNIQUE INDEX unx1_metadata_table_property + ON metadata_table_property USING BTREE (table_id, "key", is_delete); + diff --git a/dinky-admin/src/main/resources/mapper/AlertRulesMapper.xml b/dinky-admin/src/main/resources/mapper/AlertRulesMapper.xml index 99764c9680..eda2b91f9e 100644 --- a/dinky-admin/src/main/resources/mapper/AlertRulesMapper.xml +++ b/dinky-admin/src/main/resources/mapper/AlertRulesMapper.xml @@ -16,7 +16,7 @@ dat.template_content from dinky_alert_rules dar left join dinky_alert_template dat on dar.template_id = dat.id - where dar.enabled = 1 + where dar.enabled = 1 and dar.is_delete = false diff --git a/dinky-admin/src/main/resources/mapper/ClusterInstanceMapper.xml b/dinky-admin/src/main/resources/mapper/ClusterInstanceMapper.xml index d4ab493196..7019e393eb 100644 --- a/dinky-admin/src/main/resources/mapper/ClusterInstanceMapper.xml +++ b/dinky-admin/src/main/resources/mapper/ClusterInstanceMapper.xml @@ -63,7 +63,7 @@ a.* from dinky_cluster a - where enabled = 1 + where enabled = 1 and is_delete = false and `type` in( 'standalone' , 'yarn-session' , 'kubernetes-session') diff --git a/dinky-admin/src/main/resources/mapper/JobHistoryMapper.xml b/dinky-admin/src/main/resources/mapper/JobHistoryMapper.xml index 2ad1c75dc1..c56f472cb6 100644 --- a/dinky-admin/src/main/resources/mapper/JobHistoryMapper.xml +++ b/dinky-admin/src/main/resources/mapper/JobHistoryMapper.xml @@ -5,7 +5,7 @@ diff --git a/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml b/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml index a5ee9ed2dd..88b907d343 100644 --- a/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml +++ b/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml @@ -5,7 +5,7 @@ @@ -69,9 +69,10 @@ from dinky_job_instance a inner join ( select max(ji.id) as id - from dinky_job_instance ji + from dinky_job_instance ji where ji.is_delete=false group by ji.task_id ) snap on snap.id = a.id + where a.is_delete = false group by status @@ -79,28 +80,30 @@ select sum(case when b.batch then 1 else 0 end ) as batchJobCount, sum(case when b.batch then 0 else 1 end ) as streamingJobCount from (select batch_model as batch - from (select max(ji.id) as id from dinky_job_instance ji group by ji.task_id) as a + from (select max(ji.id) as id from dinky_job_instance ji where ji.is_delete=false group by ji.task_id) as a left join dinky_job_instance dji on dji.id = a.id - left join dinky_history dh on dji.history_id = dh.id) b + left join dinky_history dh on dji.history_id = dh.id + where dji.is_delete=false and dh.is_delete=false + ) b diff --git a/dinky-admin/src/main/resources/mapper/MenuMapper.xml b/dinky-admin/src/main/resources/mapper/MenuMapper.xml index 4a7cbe6196..4c250d66f7 100644 --- a/dinky-admin/src/main/resources/mapper/MenuMapper.xml +++ b/dinky-admin/src/main/resources/mapper/MenuMapper.xml @@ -23,7 +23,7 @@ select id, name, parent_id, order_num, path, component, type, display, ifnull(perms,'') as perms, icon, create_time, update_time - from dinky_sys_menu + from dinky_sys_menu where is_delete = false - - @@ -58,7 +45,7 @@ left join dinky_sys_role_menu rm on m.id = rm.menu_id left join dinky_user_role ur on rm.role_id = ur.role_id left join dinky_role ro on ur.role_id = ro.role_id - where ur.user_id = #{params.userId} + where ur.user_id = #{params.userId} and m.is_delete = false and rm.is_delete = false and ur.is_delete = false and ro.is_delete = false AND m.`name` like concat('%', #{name}, '%') @@ -74,54 +61,34 @@ left join dinky_sys_role_menu rm on m.id = rm.menu_id left join dinky_user_role ur on rm.role_id = ur.role_id left join dinky_role ro on ur.role_id = ro.role_id - left join sys_user u on ur.user_id = u.user_id - where u.user_id = #{userId} + left join dinky_user u on ur.user_id = u.id + where u.id = #{userId} and m.type in ('0', '1') - AND ro.is_delete = 0 + AND ro.is_delete = false and m.is_delete = false and rm.is_delete = false and ur.is_delete = false and ro.is_delete = false and u.is_delete = false order by m.parent_id, m.order_num - - - - - select a.* @@ -54,53 +43,16 @@ - - - \ No newline at end of file diff --git a/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml b/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml index d0b18cbd65..5b5aa96a94 100644 --- a/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml +++ b/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml @@ -11,18 +11,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" - - delete from dinky_sys_role_menu where role_id=#{roleId} - - - - insert into dinky_sys_role_menu(role_id, menu_id) values - - (#{item.roleId},#{item.menuId}) - - \ No newline at end of file diff --git a/dinky-admin/src/main/resources/mapper/TaskMapper.xml b/dinky-admin/src/main/resources/mapper/TaskMapper.xml index 031633ab65..7aa88b2edf 100644 --- a/dinky-admin/src/main/resources/mapper/TaskMapper.xml +++ b/dinky-admin/src/main/resources/mapper/TaskMapper.xml @@ -51,7 +51,7 @@ @@ -59,39 +59,9 @@ - @@ -114,7 +84,7 @@ from (select case a.batch_model when 0 then 'streaming' when 1 then 'batch' end as batch_model, coalesce(count(1), 0) as cnt from dinky_task a - where a.batch_model is not null + where a.batch_model is not null and a.enabled = 1 and a.is_delete = false and a.dialect in ( 'FlinkSql', 'FlinkJar', diff --git a/dinky-admin/src/main/resources/mapper/TenantMapper.xml b/dinky-admin/src/main/resources/mapper/TenantMapper.xml index 5ea158c7fa..0e7ae54efb 100644 --- a/dinky-admin/src/main/resources/mapper/TenantMapper.xml +++ b/dinky-admin/src/main/resources/mapper/TenantMapper.xml @@ -28,18 +28,4 @@ - \ No newline at end of file diff --git a/dinky-admin/src/main/resources/mapper/UserMapper.xml b/dinky-admin/src/main/resources/mapper/UserMapper.xml index c2ad372028..f8ec667a93 100644 --- a/dinky-admin/src/main/resources/mapper/UserMapper.xml +++ b/dinky-admin/src/main/resources/mapper/UserMapper.xml @@ -33,20 +33,6 @@ - - update dinky_user set is_delete = 0 where id = #{id} diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.14/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.14/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index 12da5b2767..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.14/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.14/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,79 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -179,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -235,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -253,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -265,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -294,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -306,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -327,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -343,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -357,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -366,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -381,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -391,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -409,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -417,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -430,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -455,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -478,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -499,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -512,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -528,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -542,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -558,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -585,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -605,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -632,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -669,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -677,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -694,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -722,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -730,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -761,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -778,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -807,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -838,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -847,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -887,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -905,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -916,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -930,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -941,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -957,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -973,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1016,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1032,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1040,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1056,20 +1064,21 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1086,7 +1095,7 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1101,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1147,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.15/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.15/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index b1824877a3..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.15/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.15/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,77 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -177,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -233,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -251,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -263,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -292,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -304,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -325,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -341,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -355,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -364,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -379,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -389,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -407,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -415,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -428,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -453,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -476,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -497,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -510,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -526,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -540,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -556,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -583,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -603,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -630,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -667,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -675,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -692,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -720,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -728,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -759,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -776,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -805,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -836,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -845,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -885,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -903,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -914,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -928,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -939,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -955,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -971,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1014,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1030,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1038,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1054,38 +1064,38 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { throw new TableNotExistException(getName(), tablePath); } - /*if (!isPartitionedTable(tablePath)) { - CatalogTableStatistics result = tableStats.get(tablePath); - return result != null ? result.copy() : CatalogTableStatistics.UNKNOWN; - } else { - return CatalogTableStatistics.UNKNOWN; - }*/ + /* + * if (!isPartitionedTable(tablePath)) { CatalogTableStatistics result = tableStats.get(tablePath); return + * result != null ? result.copy() : CatalogTableStatistics.UNKNOWN; } else { return + * CatalogTableStatistics.UNKNOWN; } + */ return CatalogTableStatistics.UNKNOWN; } @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1100,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1146,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.16/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.16/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index 12da5b2767..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.16/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.16/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,79 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -179,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -235,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -253,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -265,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -294,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -306,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -327,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -343,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -357,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -366,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -381,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -391,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -409,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -417,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -430,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -455,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -478,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -499,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -512,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -528,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -542,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -558,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -585,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -605,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -632,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -669,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -677,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -694,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -722,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -730,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -761,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -778,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -807,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -838,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -847,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -887,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -905,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -916,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -930,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -941,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -957,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -973,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1016,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1032,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1040,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1056,20 +1064,21 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1086,7 +1095,7 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1101,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1147,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.17/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.17/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index 12da5b2767..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.17/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.17/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,79 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -179,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -235,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -253,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -265,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -294,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -306,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -327,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -343,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -357,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -366,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -381,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -391,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -409,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -417,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -430,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -455,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -478,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -499,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -512,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -528,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -542,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -558,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -585,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -605,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -632,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -669,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -677,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -694,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -722,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -730,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -761,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -778,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -807,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -838,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -847,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -887,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -905,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -916,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -930,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -941,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -957,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -973,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1016,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1032,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1040,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1056,20 +1064,21 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1086,7 +1095,7 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1101,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1147,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.18/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.18/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index 12da5b2767..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.18/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.18/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,79 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -179,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -235,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -253,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -265,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -294,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -306,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -327,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -343,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -357,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -366,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -381,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -391,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -409,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -417,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -430,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -455,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -478,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -499,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -512,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -528,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -542,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -558,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -585,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -605,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -632,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -669,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -677,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -694,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -722,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -730,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -761,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -778,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -807,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -838,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -847,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -887,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -905,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -916,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -930,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -941,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -957,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -973,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1016,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1032,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1040,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1056,20 +1064,21 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1086,7 +1095,7 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1101,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1147,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.19/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.19/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index 12da5b2767..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.19/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.19/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,79 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -179,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -235,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -253,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -265,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -294,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -306,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -327,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -343,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -357,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -366,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -381,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -391,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -409,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -417,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -430,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -455,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -478,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -499,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -512,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -528,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -542,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -558,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -585,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -605,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -632,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -669,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -677,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -694,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -722,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -730,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -761,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -778,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -807,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -838,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -847,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -887,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -905,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -916,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -930,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -941,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -957,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -973,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1016,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1032,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1040,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1056,20 +1064,21 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1086,7 +1095,7 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1101,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1147,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.20/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.20/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java index 12da5b2767..83b7835b52 100644 --- a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.20/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java +++ b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.20/src/main/java/org/dinky/flink/catalog/DinkyMysqlCatalog.java @@ -73,9 +73,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Getter; + /** - * 自定义 catalog 检查connection done. 默认db，会被强制指定，不管输入的是什么，都会指定为 default_database - * 可以读取配置文件信息来获取数据库连接，而不是在sql语句中强制指定。 + * DinkyMysqlCatalog is a catalog implementation for MySQL. */ public class DinkyMysqlCatalog extends AbstractCatalog { @@ -89,79 +90,87 @@ public class DinkyMysqlCatalog extends AbstractCatalog { try { Class.forName(MYSQL_DRIVER); } catch (ClassNotFoundException e) { - throw new CatalogException("未加载 mysql 驱动！", e); + throw new CatalogException("Failed to load MySQL driver: " + MYSQL_DRIVER, e); } } private static final String COMMENT = "comment"; - /** 判断是否发生过SQL异常，如果发生过，那么conn可能失效。要注意判断 */ + /** Determine whether an SQL exception has occurred, and if so, conn may fail. Pay attention to judgment + */ private boolean sqlExceptionHappened = false; - /** 对象类型，例如 库、表、视图等 */ + /** Object types, such as libraries, tables, views, etc. + */ protected static class ObjectType { - /** 数据库 */ + /** + * Database + * */ public static final String DATABASE = "database"; - /** 数据表 */ + /** + * Table + * */ public static final String TABLE = "TABLE"; - /** 视图 */ + /** View */ public static final String VIEW = "VIEW"; } - /** 对象类型，例如 库、表、视图等 */ + /** + * + * object type, such as library, table, view, etc. + * + * */ protected static class ColumnType { - /** 物理字段 */ + /** physical field + */ public static final String PHYSICAL = "physical"; - /** 计算字段 */ + /** computed field + */ public static final String COMPUTED = "computed"; - /** 元数据字段 */ + /** + * metadata field + * */ public static final String METADATA = "metadata"; - /** 水印 */ + /** + * watermark field + * + * */ public static final String WATERMARK = "watermark"; } - /** 数据库用户名 */ - private final String user; - /** 数据库密码 */ - private final String pwd; - /** 数据库连接 */ - private final String url; - - /** 默认database */ - private static final String defaultDatabase = "default_database"; - - /** - * 数据库用户名 + /** 数据库用户名 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUser() { - return user; - } - - /** - * 数据库密码 + @Getter + private final String user; + /** 数据库密码 + * -- GETTER -- + * 数据库密码 * * @return 数据库密码 */ - public String getPwd() { - return pwd; - } - - /** - * 数据库用户名 + @Getter + private final String pwd; + /** 数据库连接 + * -- GETTER -- + * 数据库用户名 * * @return 数据库用户名 */ - public String getUrl() { - return url; - } + @Getter + private final String url; + + /** 默认database */ + private static final String defaultDatabase = "default_database"; public DinkyMysqlCatalog(String name, String url, String user, String pwd) { super(name, defaultDatabase); @@ -179,14 +188,13 @@ public DinkyMysqlCatalog(String name) { @Override public void open() throws CatalogException { - // 验证连接是否有效 - // 获取默认db看看是否存在 + Integer defaultDbId = getDatabaseId(defaultDatabase); if (defaultDbId == null) { try { createDatabase(defaultDatabase, new CatalogDatabaseImpl(new HashMap<>(), ""), true); } catch (DatabaseAlreadyExistException a) { - logger.info("重复创建默认库"); + logger.info("The default database already exists"); } } } @@ -235,7 +243,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -253,7 +261,8 @@ public List listDatabases() throws CatalogException { @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name,description FROM metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -265,7 +274,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? "; + String sql = + "select `key`,`value` from metadata_database_property where database_id=? and is_delete = 0"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -294,7 +304,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { } private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "select id from metadata_database where database_name=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -306,13 +316,13 @@ private Integer getDatabaseId(String databaseName) throws CatalogException { id = rs.getInt(1); multiDB = true; } else { - throw new CatalogException("存在多个同名database: " + databaseName); + throw new CatalogException("the database name is not unique"); } } return id; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException(String.format("获取 database 信息失败：%s.%s", getName(), databaseName), e); + throw new CatalogException(String.format("get database info fail：%s.%s", getName(), databaseName), e); } } @@ -327,9 +337,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno throw new DatabaseAlreadyExistException(getName(), databaseName); } } else { - // 在这里实现创建库的代码 Connection conn = getConnection(); - // 启动事务 String insertSql = "insert into metadata_database(database_name, description) values(?, ?)"; try (PreparedStatement stat = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS)) { @@ -343,7 +351,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno && db.getProperties().size() > 0) { int id = idRs.getInt(1); String propInsertSql = - "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)"; + "insert into metadata_database_property(database_id, `key`,`value`) values (?,?,?)"; PreparedStatement pstat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : db.getProperties().entrySet()) { pstat.setInt(1, id); @@ -357,7 +365,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - logger.error("创建 database 信息失败：", e); + logger.error("create database info failed：", e); } } } @@ -366,9 +374,9 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以删除"); + throw new CatalogException("the default database can't be deleted"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -381,7 +389,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.setAutoCommit(false); // 查询是否有表 List tables = listTables(name); - if (tables.size() > 0) { + if (!tables.isEmpty()) { if (!cascade) { // 有表，不做级联删除。 throw new DatabaseNotEmptyException(getName(), name); @@ -391,17 +399,16 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade try { dropTable(new ObjectPath(name, table), true); } catch (TableNotExistException t) { - logger.warn("表{}不存在", name + "." + table); + logger.warn("table {} is not exits", name + "." + table); } } } - // todo: 现在是真实删除，后续设计是否做记录保留。 - String deletePropSql = "delete from metadata_database_property where database_id=?"; + String deletePropSql = "update metadata_database set is_delete=1 where id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; + String deleteDbSql = "update metadata_database set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -409,7 +416,7 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 database 信息失败：", e); + throw new CatalogException("drop database failed：", e); } } @@ -417,9 +424,9 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNotExists) throws DatabaseNotExistException, CatalogException { if (name.equals(defaultDatabase)) { - throw new CatalogException("默认 database 不可以修改"); + throw new CatalogException("the default database can't be altered"); } - // 1、取出db id， + Integer id = getDatabaseId(name); if (id == null) { if (!ignoreIfNotExists) { @@ -430,8 +437,8 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo Connection conn = getConnection(); try { conn.setAutoCommit(false); - // 1、名称不能改，类型不能改。只能改备注 - String updateCommentSql = "update metadata_database set description=? where id=?"; + + String updateCommentSql = "update metadata_database set description=? where id=? and is_delete = 0"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); @@ -455,7 +462,7 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo conn.commit(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 database 信息失败：", e); + throw new CatalogException("alert database info failed：", e); } } @@ -478,7 +485,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // 要给出table 或 view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table where table_type=? and database_id = ? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -499,10 +507,6 @@ protected List listTablesViews(String databaseName, String tableType) @Override public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // 还是分步骤来 - // 1、先取出表 这可能是view也可能是table - // 2、取出列 - // 3、取出属性 Integer id = getTableId(tablePath); if (id == null) { @@ -512,7 +516,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { String queryTable = - "SELECT table_name " + " ,description, table_type " + " FROM metadata_table " + " where id=?"; + "SELECT table_name ,description, table_type FROM metadata_table where id=? and is_delete = 0"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -528,7 +532,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // 这个是 table - String propSql = "SELECT `key`, `value` from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT `key`, `value` from metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -542,11 +547,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep props.put(COMMENT, description); return CatalogTable.fromProperties(props); } else if (tableType.equals(ObjectType.VIEW)) { - // 1、从库中取出table信息。（前面已做） - // 2、取出字段。 + String colSql = "SELECT column_name, column_type, data_type, description " + " FROM metadata_column WHERE " - + " table_id=?"; + + " table_id=? and is_delete = 0"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -558,13 +562,13 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep builder.column(colName, dataType); String cDesc = crs.getString("description"); - if (null != cDesc && cDesc.length() > 0) { + if (null != cDesc && !cDesc.isEmpty()) { builder.withComment(cDesc); } } cStat.close(); // 3、取出query - String qSql = "SELECT `key`, value FROM metadata_table_property" + " WHERE table_id=? "; + String qSql = "SELECT `key`, value FROM metadata_table_property WHERE table_id=? and is_delete = 0"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -585,11 +589,11 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 合成view return CatalogView.of(builder.build(), description, originalQuery, expandedQuery, options); } else { - throw new CatalogException("不支持的数据类型。" + tableType); + throw new CatalogException("an unsupported data type。" + tableType); } } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 表信息失败。", e); + throw new CatalogException("Failed to obtain table information。", e); } } @@ -605,7 +609,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table " + " where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -632,19 +636,20 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) } Connection conn = getConnection(); try { - // todo: 现在是真实删除，后续设计是否做记录保留。 conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property " + " where table_id=?"; + + String deletePropSql = "update metadata_table_property set is_delete=1 where table_id=?"; PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column " + " where table_id=?"; + String deleteColSql = "update metadata_column set is_delete=1 where table_id=?"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table " + " where id=?"; + // String deleteDbSql = "delete from metadata_table " + " where id=?"; + String deleteDbSql = "update metadata_table set is_delete=1 where id=?"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -669,7 +674,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -677,7 +682,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor ps.executeUpdate(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } @@ -694,12 +699,14 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } return; } - // 插入表 - // 插入到table表。这里，它可能是table也可能是view - // 如果是一个table，我们认为它是一个 resolved table，就可以使用properties方式来进行序列化并保存。 - // 如果是一个view，我们认为它只能有物理字段 + // Insert table + // Insert into the table table. Here, it could be a table or a view + // If it is a table, we think it is a resolved table, so we can use the properties method to serialize and save + // it. + // If it is a view, we think it can only have physical fields if (!(table instanceof ResolvedCatalogBaseTable)) { - throw new UnsupportedOperationException("暂时不支持输入非 ResolvedCatalogBaseTable 类型的表"); + throw new UnsupportedOperationException( + "Entering tables of non-ResolvedCatalogBaseTable types is temporarily not supported"); } Connection conn = getConnection(); try { @@ -722,7 +729,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig ResultSet idRs = iStat.getGeneratedKeys(); if (!idRs.next()) { iStat.close(); - throw new CatalogException("插入元数据表信息失败"); + throw new CatalogException("Failed to insert metadata table information"); } int id = idRs.getInt(1); iStat.close(); @@ -730,8 +737,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (table instanceof ResolvedCatalogTable) { // table 就可以直接拿properties了。 Map props = ((ResolvedCatalogTable) table).toProperties(); - String propInsertSql = - "insert into metadata_table_property(table_id," + "`key`,`value`) values (?,?,?)"; + String propInsertSql = "insert into metadata_table_property(table_id, `key`,`value`) values (?,?,?)"; PreparedStatement pStat = conn.prepareStatement(propInsertSql); for (Map.Entry entry : props.entrySet()) { pStat.setInt(1, id); @@ -761,7 +767,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig Schema.UnresolvedPhysicalColumn pCol = (Schema.UnresolvedPhysicalColumn) col; if (!(pCol.getDataType() instanceof DataType)) { throw new UnsupportedOperationException(String.format( - "类型识别失败，该列不是有效类型：%s.%s.%s : %s", + "Type identification failed. The column is not a valid type：%s.%s.%s : %s", tablePath.getDatabaseName(), tablePath.getObjectName(), pCol.getName(), @@ -778,7 +784,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig colIStat.setObject(7, null); // view没有主键 colIStat.addBatch(); } else { - throw new UnsupportedOperationException("暂时认为view 不会出现 非物理字段"); + throw new UnsupportedOperationException( + "Temporarily, it is believed that non-physical fields will not appear in view"); } } colIStat.executeBatch(); @@ -807,8 +814,8 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig conn.commit(); } catch (SQLException ex) { sqlExceptionHappened = true; - logger.error("插入数据库失败", ex); - throw new CatalogException("插入数据库失败", ex); + logger.error("Failed to insert database\n", ex); + throw new CatalogException("Failed to insert database\n", ex); } } @@ -838,7 +845,7 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ps.executeBatch(); } catch (SQLException ex) { sqlExceptionHappened = true; - throw new CatalogException("修改表名失败", ex); + throw new CatalogException("Failed to modify table name", ex); } } } @@ -847,36 +854,36 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean @Override public List listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public List listPartitionsByFilter(ObjectPath tablePath, List filters) throws TableNotExistException, TableNotPartitionedException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -887,15 +894,15 @@ public void createPartition( boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } @Override @@ -905,8 +912,8 @@ public void alterPartition( CatalogPartition newPartition, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。 + throw new UnsupportedOperationException("The method is not yet complete\n"); } /** *********************Functions********************* */ @@ -916,7 +923,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function " + " WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -930,7 +937,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio return functions; } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("获取 UDF 列表失败"); + throw new CatalogException("Failed to get UDF list", e); } } @@ -941,7 +948,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function " + " WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -957,7 +964,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx } catch (SQLException e) { sqlExceptionHappened = true; throw new CatalogException( - "获取 UDF 失败：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); + "Failed to get UDF list：" + functionPath.getDatabaseName() + "." + functionPath.getObjectName()); } } @@ -973,7 +980,7 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // 获取id - String getIdSql = "select id from metadata_function " + " where function_name=? and database_id=?"; + String getIdSql = "select id from metadata_function where function_name=? and database_id=? and is_delete = 0"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1016,7 +1023,7 @@ public void createFunction(ObjectPath functionPath, CatalogFunction function, bo ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("创建 函数 失败", e); + throw new CatalogException("Failed to create function", e); } } @@ -1032,7 +1039,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function " + "set (class_name =?, function_language=?) " + " where id=?"; + String insertSql = + "update metadata_function set (class_name =?, function_language=?) " + " where id=? and is_delete = 0"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1040,7 +1048,7 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("修改 函数 失败", e); + throw new CatalogException("Failed to modify function", e); } } @@ -1056,20 +1064,21 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function " + " where id=?"; + // String insertSql = "delete from metadata_function " + " where id=?"; + String insertSql = "update metadata_function set is_delete = 1 where id=?"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); } catch (SQLException e) { sqlExceptionHappened = true; - throw new CatalogException("删除 函数 失败", e); + throw new CatalogException("drop function fail.", e); } } @Override public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1086,7 +1095,7 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) @Override public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 + // todo: The method is not yet complete。。 checkNotNull(tablePath); if (!tableExists(tablePath)) { @@ -1101,32 +1110,32 @@ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) @Override public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public CatalogColumnStatistics getPartitionColumnStatistics( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override public void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException, TablePartitionedException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } @Override @@ -1147,7 +1156,7 @@ public void alterPartitionColumnStatistics( CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException { - // todo: 补充完成该方法。 - throw new UnsupportedOperationException("该方法尚未完成"); + // todo: The method is not yet complete。。 + throw new UnsupportedOperationException("The method is not yet complete。"); } } diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.14/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.14/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.14/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.14/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.15/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.15/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.15/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.15/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.16/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.16/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.16/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.16/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.17/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.17/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.17/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.17/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.18/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.18/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.18/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.18/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.19/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.19/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.19/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.19/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate(); diff --git a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.20/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.20/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java index 5357b49a70..382102b7ff 100644 --- a/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.20/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java +++ b/dinky-catalog/dinky-catalog-postgres/dinky-catalog-postgres-1.20/src/main/java/org/dinky/flink/catalog/DinkyPostgresCatalog.java @@ -276,7 +276,7 @@ protected Connection getConnection() throws CatalogException { @Override public List listDatabases() throws CatalogException { List myDatabases = new ArrayList<>(); - String querySql = "SELECT database_name FROM metadata_database"; + String querySql = "SELECT database_name FROM metadata_database WHERE is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { @@ -302,7 +302,8 @@ public List listDatabases() throws CatalogException { */ @Override public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { - String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?"; + String querySql = + "SELECT id, database_name, description FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -314,7 +315,8 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE Map map = new HashMap<>(); - String sql = "select \"key\", \"value\" from metadata_database_property where database_id=? "; + String sql = + "SELECT \"key\", \"value\" FROM metadata_database_property WHERE database_id=? AND is_delete=false"; try (PreparedStatement pStat = conn.prepareStatement(sql)) { pStat.setInt(1, id); ResultSet prs = pStat.executeQuery(); @@ -357,7 +359,7 @@ public boolean databaseExists(String databaseName) throws CatalogException { * @throws CatalogException This exception is thrown if an error occurs while getting the database ID */ private Integer getDatabaseId(String databaseName) throws CatalogException { - String querySql = "select id from metadata_database where database_name=?"; + String querySql = "SELECT id FROM metadata_database WHERE database_name=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, databaseName); @@ -479,13 +481,14 @@ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade } } // todo: Now it is actually deleted, whether records will be retained for subsequent designs. - String deletePropSql = "delete from metadata_database_property where database_id=?"; - PreparedStatement dStat = conn.prepareStatement(deletePropSql); + String updatePropSql = + "UPDATE metadata_database_property SET is_delete=true WHERE database_id=? AND is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(updatePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_database where id=?"; - dStat = conn.prepareStatement(deleteDbSql); + String updateDbSql = "UPDATE metadata_database SET is_delete=true WHERE id=? AND is_delete=false"; + dStat = conn.prepareStatement(updateDbSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); @@ -523,16 +526,17 @@ public void alterDatabase(String name, CatalogDatabase newDb, boolean ignoreIfNo try { conn.setAutoCommit(false); // 1. The name cannot be changed and the type cannot be changed. Only notes can be changed - String updateCommentSql = "update metadata_database set description=? where id=?"; + String updateCommentSql = "UPDATE metadata_database SET description=? WHERE id=? AND is_delete=false"; PreparedStatement uState = conn.prepareStatement(updateCommentSql); uState.setString(1, newDb.getComment()); uState.setInt(2, id); uState.executeUpdate(); uState.close(); if (newDb.getProperties() != null && newDb.getProperties().size() > 0) { - String upsertSql = "insert into metadata_database_property (database_id, key, value) " - + "values (?,?,?) " - + "on CONFLICT (database_id, \"key\") do update set value = excluded.value, update_time = now()"; + String upsertSql = "INSERT INTO metadata_database_property (database_id, key, value) " + + "VALUES (?, ?, ?) " + + "ON CONFLICT (database_id, \"key\") DO UPDATE SET value = excluded.value, update_time = now() " + + "WHERE is_delete=false"; PreparedStatement pstat = conn.prepareStatement(upsertSql); for (Map.Entry entry : newDb.getProperties().entrySet()) { pstat.setInt(1, id); @@ -593,7 +597,8 @@ protected List listTablesViews(String databaseName, String tableType) // get all schemas // To give table or view - String querySql = "SELECT table_name FROM metadata_table where table_type=? and database_id = ?"; + String querySql = + "SELECT table_name FROM metadata_table WHERE database_name=? AND object_type=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(querySql)) { ps.setString(1, tableType); @@ -634,7 +639,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep Connection conn = getConnection(); try { - String queryTable = "SELECT table_name ,description, table_type FROM metadata_table where id=?"; + String queryTable = + "SELECT table_name ,description, table_type FROM metadata_table where id=? AND is_delete=false"; PreparedStatement ps = conn.prepareStatement(queryTable); ps.setInt(1, id); ResultSet rs = ps.executeQuery(); @@ -650,7 +656,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } if (tableType.equals(ObjectType.TABLE)) { // This is table - String propSql = "SELECT \"key\", \"value\" from metadata_table_property " + "WHERE table_id=?"; + String propSql = + "SELECT \"key\", \"value\" from metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement pState = conn.prepareStatement(propSql); pState.setInt(1, id); ResultSet prs = pState.executeQuery(); @@ -667,7 +674,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep // 1. Get table information from the library. (Already done before) // 2. Take out the field. String colSql = - "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=?"; + "SELECT column_name, column_type, data_type, description FROM metadata_column WHERE table_id=? AND is_delete=false"; PreparedStatement cStat = conn.prepareStatement(colSql); cStat.setInt(1, id); ResultSet crs = cStat.executeQuery(); @@ -685,7 +692,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } cStat.close(); // 3、Take out the query - String qSql = "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? "; + String qSql = + "SELECT \"key\", \"value\" FROM metadata_table_property WHERE table_id=? AND is_delete=false"; PreparedStatement qStat = conn.prepareStatement(qSql); qStat.setInt(1, id); ResultSet qrs = qStat.executeQuery(); @@ -740,7 +748,7 @@ private Integer getTableId(ObjectPath tablePath) { return null; } // 获取id - String getIdSql = "select id from metadata_table where table_name=? and database_id=?"; + String getIdSql = "select id from metadata_table where table_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, tablePath.getObjectName()); @@ -777,17 +785,22 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) try { // todo: Now it is actually deleted, whether records will be retained for subsequent designs. conn.setAutoCommit(false); - String deletePropSql = "delete from metadata_table_property where table_id=?"; + // String deletePropSql = "delete from metadata_table_property where table_id=?"; + String deletePropSql = + "UPDATE metadata_table_property SET is_delete=true WHERE table_id=? and is_delete=false"; + PreparedStatement dStat = conn.prepareStatement(deletePropSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteColSql = "delete from metadata_column where table_id=?"; + // String deleteColSql = "delete from metadata_column where table_id=?"; + String deleteColSql = "UPDATE metadata_column SET is_delete=true WHERE table_id=? and is_delete=false"; dStat = conn.prepareStatement(deleteColSql); dStat.setInt(1, id); dStat.executeUpdate(); dStat.close(); - String deleteDbSql = "delete from metadata_table where id=?"; + // String deleteDbSql = "delete from metadata_table where id=?"; + String deleteDbSql = "UPDATE metadata_table SET is_delete=true WHERE id=? and is_delete=false"; dStat = conn.prepareStatement(deleteDbSql); dStat.setInt(1, id); dStat.executeUpdate(); @@ -822,7 +835,7 @@ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignor if (tableExists(newPath)) { throw new TableAlreadyExistException(getName(), newPath); } - String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=?"; + String updateSql = "UPDATE metadata_table SET table_name=? WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { ps.setString(1, newTableName); @@ -999,7 +1012,9 @@ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean String updateSql = "INSERT INTO metadata_table_property(table_id," + "key, value) values (?,?,?) " - + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()"; + + "on CONFLICT (table_id, \"key\") do update set value = excluded.value, update_time = now()" + + "WHERE is_delete=false"; + Connection conn = getConnection(); try (PreparedStatement ps = conn.prepareStatement(updateSql)) { for (Map.Entry entry : opts.entrySet()) { @@ -1178,7 +1193,7 @@ public List listFunctions(String dbName) throws DatabaseNotExistExceptio if (null == dbId) { throw new DatabaseNotExistException(getName(), dbName); } - String querySql = "SELECT function_name from metadata_function WHERE database_id=?"; + String querySql = "SELECT function_name from metadata_function WHERE database_id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { @@ -1211,7 +1226,7 @@ public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotEx throw new FunctionNotExistException(getName(), functionPath); } - String querySql = "SELECT class_name,function_language from metadata_function WHERE id=?"; + String querySql = "SELECT class_name,function_language from metadata_function WHERE id=? AND is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(querySql)) { gStat.setInt(1, id); @@ -1256,7 +1271,8 @@ private Integer getFunctionId(ObjectPath functionPath) { return null; } // Get id - String getIdSql = "select id from metadata_function where function_name=? and database_id=?"; + String getIdSql = + "select id from metadata_function where function_name=? and database_id=? and is_delete=false"; Connection conn = getConnection(); try (PreparedStatement gStat = conn.prepareStatement(getIdSql)) { gStat.setString(1, functionPath.getObjectName()); @@ -1334,7 +1350,8 @@ public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, } Connection conn = getConnection(); - String insertSql = "update metadata_function set class_name =?, function_language=? where id=?"; + String insertSql = + "update metadata_function set class_name =?, function_language=? where id=? and is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, newFunction.getClassName()); ps.setString(2, newFunction.getFunctionLanguage().toString()); @@ -1366,7 +1383,8 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) } Connection conn = getConnection(); - String insertSql = "delete from metadata_function where id=?"; + // String insertSql = "delete from metadata_function where id=?"; + String insertSql = "UPDATE metadata_function SET is_delete=true WHERE id=? AND is_delete=false"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setInt(1, id); ps.executeUpdate();