Skip to content

Commit

Permalink
[opt](binlog) Support drop view binlog
Browse files Browse the repository at this point in the history
  • Loading branch information
@smallx @wyxxxcat
smallx authored and wyxxxcat committed Nov 9, 2024
1 parent a843ca3 commit d5b5fc0
Show file tree
Hide file tree
Showing 7 changed files with 52 additions and 37 deletions.
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -935,7 +935,8 @@ public void processDropMaterializedView(DropMaterializedViewStmt dropMaterialize
// Step3: log drop mv operation
EditLog editLog = Env.getCurrentEnv().getEditLog();
editLog.logDropRollup(
new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, 0));
new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, false, 0));
deleteIndexList.add(mvIndexId);
LOG.info("finished drop materialized view [{}] in table [{}]", mvName, olapTable.getName());
} catch (MetaNotFoundException e) {
if (dropMaterializedViewStmt.isIfExists()) {
Expand Down
31 changes: 13 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2054,29 +2054,24 @@ private Status allTabletCommitted(boolean isReplay) {
}

private Status dropAllNonRestoredTableAndPartitions(Database db) {
Set<String> restoredViews = jobInfo.newBackupObjects.views.stream()
.map(view -> view.name).collect(Collectors.toSet());

try {
for (Table table : db.getTables()) {
long tableId = table.getId();
String tableName = table.getName();
TableType tableType = table.getType();
if (tableType == TableType.OLAP) {
BackupOlapTableInfo backupTableInfo = jobInfo.backupOlapTableObjects.get(tableName);
if (tableType == TableType.OLAP && backupTableInfo != null) {
// drop the non restored partitions.
dropNonRestoredPartitions(db, (OlapTable) table, backupTableInfo);
} else if (isCleanTables) {
// otherwise drop the entire table.
LOG.info("drop non restored table {}, table id: {}. {}", tableName, tableId, this);
boolean isForceDrop = false; // move this table into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop);
}
} else if (tableType == TableType.VIEW && isCleanTables && !restoredViews.contains(tableName)) {
LOG.info("drop non restored view {}, table id: {}. {}", tableName, tableId, this);
boolean isForceDrop = false; // move this view into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop);
BackupOlapTableInfo backupTableInfo = jobInfo.backupOlapTableObjects.get(tableName);
if (tableType != TableType.OLAP && tableType != TableType.ODBC && tableType != TableType.VIEW) {
continue;
}
if (tableType == TableType.OLAP && backupTableInfo != null) {
// drop the non restored partitions.
dropNonRestoredPartitions(db, (OlapTable) table, backupTableInfo);
} else if (isCleanTables) {
// otherwise drop the entire table.
LOG.info("drop non restored table {}({}). {}", tableName, tableId, this);
boolean isView = false;
boolean isForceDrop = false; // move this table into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isView, isForceDrop);
}
}
return Status.OK;
Expand Down
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ public class DropTableRecord {
private long tableId;
@SerializedName(value = "tableName")
private String tableName;
@SerializedName(value = "isView")
private boolean isView = false;
@SerializedName(value = "rawSql")
private String rawSql;

Expand All @@ -39,7 +41,10 @@ public DropTableRecord(long commitSeq, DropInfo info) {
this.dbId = info.getDbId();
this.tableId = info.getTableId();
this.tableName = info.getTableName();
this.rawSql = String.format("DROP TABLE IF EXISTS `%s`", this.tableName);
this.isView = info.isView();
this.rawSql = info.isView()
? String.format("DROP VIEW IF EXISTS `%s`", this.tableName)
: String.format("DROP TABLE IF EXISTS `%s`", this.tableName);
}

public long getCommitSeq() {
Expand Down
20 changes: 13 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -916,23 +916,26 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
}
}

dropTableInternal(db, table, stmt.isForceDrop());
dropTableInternal(db, table, stmt.isView(), stmt.isForceDrop(), watch, costTimes);
} catch (UserException e) {
throw new DdlException(e.getMessage(), e.getMysqlErrorCode());
} finally {
db.writeUnlock();
}
LOG.info("finished dropping table: {} from db: {}, is force: {}", tableName, dbName, stmt.isForceDrop());
watch.stop();
costTimes.put("6:total", watch.getTime());
LOG.info("finished dropping table: {} from db: {}, is view: {}, is force: {}, cost: {}",
tableName, dbName, stmt.isView(), stmt.isForceDrop(), costTimes);
}

// drop table without any check.
public void dropTableWithoutCheck(Database db, Table table, boolean forceDrop) throws DdlException {
public void dropTableWithoutCheck(Database db, Table table, boolean isView, boolean forceDrop) throws DdlException {
if (!db.writeLockIfExist()) {
return;
}
try {
LOG.info("drop table {} without check, force: {}", table.getQualifiedName(), forceDrop);
dropTableInternal(db, table, forceDrop);
dropTableInternal(db, table, isView, forceDrop, null, null);
} catch (Exception e) {
LOG.warn("drop table without check", e);
throw e;
Expand All @@ -942,7 +945,8 @@ public void dropTableWithoutCheck(Database db, Table table, boolean forceDrop) t
}

// Drop a table, the db lock must hold.
private void dropTableInternal(Database db, Table table, boolean forceDrop) throws DdlException {
private void dropTableInternal(Database db, Table table, boolean isView, boolean forceDrop,
StopWatch watch, Map<String, Long> costTimes) throws DdlException {
table.writeLock();
String tableName = table.getName();
long recycleTime = 0;
Expand All @@ -955,7 +959,9 @@ private void dropTableInternal(Database db, Table table, boolean forceDrop) thro
table.writeUnlock();
}

DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, forceDrop, recycleTime);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(),
db.getId(), table.getId());
DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, isView, forceDrop, recycleTime);
Env.getCurrentEnv().getEditLog().logDropTable(info);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(),
db.getId(), table.getId());
Expand Down Expand Up @@ -2732,7 +2738,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
try {
dropTable(db, tableId, true, false, 0L);
if (hadLogEditCreateTable) {
DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, true, 0L);
DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, false, true, 0L);
Env.getCurrentEnv().getEditLog().logDropTable(info);
}
} catch (Exception ex) {
Expand Down
16 changes: 12 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ public class DropInfo implements Writable {
private String tableName; // not used in equals and hashCode
@SerializedName(value = "indexId")
private long indexId;
@SerializedName(value = "isView")
private boolean isView = false;
@SerializedName(value = "forceDrop")
private boolean forceDrop = false;
@SerializedName(value = "recycleTime")
Expand All @@ -46,11 +48,13 @@ public class DropInfo implements Writable {
public DropInfo() {
}

public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean forceDrop, long recycleTime) {
public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean isView, boolean forceDrop,
long recycleTime) {
this.dbId = dbId;
this.tableId = tableId;
this.tableName = tableName;
this.indexId = indexId;
this.isView = isView;
this.forceDrop = forceDrop;
this.recycleTime = recycleTime;
}
Expand All @@ -71,12 +75,16 @@ public long getIndexId() {
return this.indexId;
}

public boolean isView() {
return this.isView;
}

public boolean isForceDrop() {
return forceDrop;
return this.forceDrop;
}

public Long getRecycleTime() {
return recycleTime;
return this.recycleTime;
}

@Override
Expand Down Expand Up @@ -119,7 +127,7 @@ public boolean equals(Object obj) {
DropInfo info = (DropInfo) obj;

return (dbId == info.dbId) && (tableId == info.tableId) && (indexId == info.indexId)
&& (forceDrop == info.forceDrop) && (recycleTime == info.recycleTime);
&& (isView == info.isView) && (forceDrop == info.forceDrop) && (recycleTime == info.recycleTime);
}

public String toJson() {
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
for (long indexId : batchDropInfo.getIndexIdSet()) {
env.getMaterializedViewHandler().replayDropRollup(
new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
batchDropInfo.getTableName(), indexId, false, 0),
batchDropInfo.getTableName(), indexId, false, false, 0),
env);
}
break;
Expand Down
10 changes: 5 additions & 5 deletions fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public void testDropInfoSerialization() throws Exception {
DropInfo info1 = new DropInfo();
info1.write(dos);

DropInfo info2 = new DropInfo(1, 2, "t2", -1, true, 0);
DropInfo info2 = new DropInfo(1, 2, "t2", -1, false, true, 0);
info2.write(dos);

dos.flush();
Expand All @@ -65,10 +65,10 @@ public void testDropInfoSerialization() throws Exception {

Assert.assertEquals(rInfo2, rInfo2);
Assert.assertNotEquals(rInfo2, this);
Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, 0));
Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, true, 0));
Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, false, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, false, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, false, 0));
Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, false, true, 0));

// 3. delete files
dis.close();
Expand Down

0 comments on commit d5b5fc0

Please sign in to comment.