diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java new file mode 100644 index 0000000000000..75511894149d4 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.flink.action.CompactDatabaseAction; +import org.apache.paimon.utils.StringUtils; +import org.apache.paimon.utils.TimeUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.util.Map; + +import static org.apache.paimon.utils.ParameterUtils.parseCommaSeparatedKeyValues; + +/** + * Compact database procedure. Usage: + * + *

+ *  -- NOTE: use '' as placeholder for optional arguments
+ *
+ *  -- compact all databases
+ *  CALL sys.compact_database()
+ *
+ *  -- compact some databases (accept regular expression)
+ *  CALL sys.compact_database('includingDatabases')
+ *
+ *  -- set compact mode
+ *  CALL sys.compact_database('includingDatabases', 'mode')
+ *
+ *  -- compact some tables (accept regular expression)
+ *  CALL sys.compact_database('includingDatabases', 'mode', 'includingTables')
+ *
+ *  -- exclude some tables (accept regular expression)
+ *  CALL sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables')
+ *
+ *  -- set table options ('k=v,...')
+ *  CALL sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables', 'tableOptions')
+ * 
+ */ +public class CompactDatabaseProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "compact_database"; + + public String[] call(ProcedureContext procedureContext) throws Exception { + return call(procedureContext, ""); + } + + public String[] call(ProcedureContext procedureContext, String includingDatabases) + throws Exception { + return call(procedureContext, includingDatabases, ""); + } + + public String[] call(ProcedureContext procedureContext, String includingDatabases, String mode) + throws Exception { + return call(procedureContext, includingDatabases, mode, ""); + } + + public String[] call( + ProcedureContext procedureContext, + String includingDatabases, + String mode, + String includingTables) + throws Exception { + return call(procedureContext, includingDatabases, mode, includingTables, ""); + } + + public String[] call( + ProcedureContext procedureContext, + String includingDatabases, + String mode, + String includingTables, + String excludingTables) + throws Exception { + return call( + procedureContext, includingDatabases, mode, includingTables, excludingTables, ""); + } + + public String[] call( + ProcedureContext procedureContext, + String includingDatabases, + String mode, + String includingTables, + String excludingTables, + String tableOptions) + throws Exception { + return call( + procedureContext, + includingDatabases, + mode, + includingTables, + excludingTables, + tableOptions, + ""); + } + + public String[] call( + ProcedureContext procedureContext, + String includingDatabases, + String mode, + String includingTables, + String excludingTables, + String tableOptions, + String partitionIdleTime) + throws Exception { + String warehouse = catalog.warehouse(); + Map catalogOptions = catalog.options(); + CompactDatabaseAction action = + new CompactDatabaseAction(warehouse, catalogOptions) + .includingDatabases(nullable(includingDatabases)) + .includingTables(nullable(includingTables)) + .excludingTables(nullable(excludingTables)) + .withDatabaseCompactMode(nullable(mode)); + if (!StringUtils.isBlank(tableOptions)) { + action.withTableOptions(parseCommaSeparatedKeyValues(tableOptions)); + } + if (!StringUtils.isBlank(partitionIdleTime)) { + action.withPartitionIdleTime(TimeUtils.parseDuration(partitionIdleTime)); + } + + return execute(procedureContext, action, "Compact database job"); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateBranchProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateBranchProcedure.java new file mode 100644 index 0000000000000..093505923fd6e --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateBranchProcedure.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.Table; + +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Create branch procedure for given tag. Usage: + * + *

+ *  CALL sys.create_branch('tableId', 'branchName', 'tagName')
+ * 
+ */ +public class CreateBranchProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "create_branch"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + public String[] call( + ProcedureContext procedureContext, String tableId, String branchName, String tagName) + throws Catalog.TableNotExistException { + return innerCall(tableId, branchName, tagName); + } + + public String[] call(ProcedureContext procedureContext, String tableId, String branchName) + throws Catalog.TableNotExistException { + return innerCall(tableId, branchName, null); + } + + private String[] innerCall(String tableId, String branchName, String tagName) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + if (!StringUtils.isBlank(tagName)) { + table.createBranch(branchName, tagName); + } else { + table.createBranch(branchName); + } + return new String[] {"Success"}; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java new file mode 100644 index 0000000000000..1a7b03ef65127 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.TimeUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import javax.annotation.Nullable; + +import java.time.Duration; + +/** + * Create tag procedure. Usage: + * + *

+ *  CALL sys.create_tag('tableId', 'tagName', snapshotId, 'timeRetained')
+ * 
+ */ +public class CreateTagProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "create_tag"; + + public String[] call( + ProcedureContext procedureContext, String tableId, String tagName, long snapshotId) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, snapshotId, null); + } + + public String[] call(ProcedureContext procedureContext, String tableId, String tagName) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, null, null); + } + + public String[] call( + ProcedureContext procedureContext, + String tableId, + String tagName, + long snapshotId, + String timeRetained) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, snapshotId, timeRetained); + } + + public String[] call( + ProcedureContext procedureContext, String tableId, String tagName, String timeRetained) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, null, timeRetained); + } + + private String[] innerCall( + String tableId, + String tagName, + @Nullable Long snapshotId, + @Nullable String timeRetained) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + if (snapshotId == null) { + table.createTag(tagName, toDuration(timeRetained)); + } else { + table.createTag(tagName, snapshotId, toDuration(timeRetained)); + } + return new String[] {"Success"}; + } + + @Nullable + private static Duration toDuration(@Nullable String s) { + if (s == null) { + return null; + } + + return TimeUtils.parseDuration(s); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MarkPartitionDoneProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MarkPartitionDoneProcedure.java new file mode 100644 index 0000000000000..d70cccf6ba25c --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MarkPartitionDoneProcedure.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.partition.actions.PartitionMarkDoneAction; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.PartitionPathUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.io.IOException; +import java.util.List; + +import static org.apache.paimon.flink.sink.partition.PartitionMarkDone.markDone; +import static org.apache.paimon.utils.ParameterUtils.getPartitions; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** + * Partition mark done procedure. Usage: + * + *

+ *  CALL sys.mark_partition_done('tableId', 'partition1', 'partition2', ...)
+ * 
+ */ +public class MarkPartitionDoneProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "mark_partition_done"; + + public String[] call( + ProcedureContext procedureContext, String tableId, String... partitionStrings) + throws Catalog.TableNotExistException, IOException { + checkArgument( + partitionStrings.length > 0, + "mark_partition_done procedure must specify partitions."); + + Identifier identifier = Identifier.fromString(tableId); + Table table = catalog.getTable(identifier); + checkArgument( + table instanceof FileStoreTable, + "Only FileStoreTable supports mark_partition_done procedure. The table type is '%s'.", + table.getClass().getName()); + + FileStoreTable fileStoreTable = (FileStoreTable) table; + CoreOptions coreOptions = fileStoreTable.coreOptions(); + List actions = + PartitionMarkDoneAction.createActions(fileStoreTable, coreOptions); + + List partitionPaths = + PartitionPathUtils.generatePartitionPaths( + getPartitions(partitionStrings), fileStoreTable.store().partitionType()); + + markDone(partitionPaths, actions); + + IOUtils.closeAllQuietly(actions); + + return new String[] {"Success"}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java new file mode 100644 index 0000000000000..acda2afd2e697 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.flink.action.MergeIntoAction; + +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.procedure.ProcedureContext; + +import java.util.Map; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** + * Merge Into procedure. Usage: + * + *

+ *  -- NOTE: use '' as placeholder for optional arguments
+ *
+ *  -- when matched then upsert
+ *  CALL sys.merge_into(
+ *      'targetTableId',
+ *      'targetAlias',
+ *      'sourceSqls', -- separate with ';'
+ *      'sourceTable',
+ *      'mergeCondition',
+ *      'matchedUpsertCondition',
+ *      'matchedUpsertSetting'
+ *  )
+ *
+ *  -- when matched then upsert + when not matched then insert
+ *  CALL sys.merge_into(
+ *      'targetTableId'
+ *      'targetAlias',
+ *      'sourceSqls',
+ *      'sourceTable',
+ *      'mergeCondition',
+ *      'matchedUpsertCondition',
+ *      'matchedUpsertSetting',
+ *      'notMatchedInsertCondition',
+ *      'notMatchedInsertValues'
+ *  )
+ *
+ *  -- above + when matched then delete
+ *  -- IMPORTANT: Use 'TRUE' if you want to delete data without filter condition.
+ *  -- If matchedDeleteCondition='', it will ignore matched-delete action!
+ *  CALL sys.merge_into(
+ *      'targetTableId',
+ *      'targetAlias',
+ *      'sourceSqls',
+ *      'sourceTable',
+ *      'mergeCondition',
+ *      'matchedUpsertCondition',
+ *      'matchedUpsertSetting',
+ *      'notMatchedInsertCondition',
+ *      'notMatchedInsertValues',
+ *      'matchedDeleteCondition'
+ *  )
+ *
+ *  -- when matched then delete (short form)
+ *  CALL sys.merge_into(
+ *      'targetTableId'
+ *      'targetAlias',
+ *      'sourceSqls',
+ *      'sourceTable',
+ *      'mergeCondition',
+ *      'matchedDeleteCondition'
+ *  )
+ * 
+ * + *

This procedure will be forced to use batch environments. Compared to {@link MergeIntoAction}, + * this procedure doesn't provide arguments to control not-matched-by-source behavior because they + * are not commonly used and will make the methods too complex to use. + */ +public class MergeIntoProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "merge_into"; + + public String[] call( + ProcedureContext procedureContext, + String targetTableId, + String targetAlias, + String sourceSqls, + String sourceTable, + String mergeCondition, + String matchedUpsertCondition, + String matchedUpsertSetting) { + return call( + procedureContext, + targetTableId, + targetAlias, + sourceSqls, + sourceTable, + mergeCondition, + matchedUpsertCondition, + matchedUpsertSetting, + "", + "", + ""); + } + + public String[] call( + ProcedureContext procedureContext, + String targetTableId, + String targetAlias, + String sourceSqls, + String sourceTable, + String mergeCondition, + String matchedUpsertCondition, + String matchedUpsertSetting, + String notMatchedInsertCondition, + String notMatchedInsertValues) { + return call( + procedureContext, + targetTableId, + targetAlias, + sourceSqls, + sourceTable, + mergeCondition, + matchedUpsertCondition, + matchedUpsertSetting, + notMatchedInsertCondition, + notMatchedInsertValues, + ""); + } + + public String[] call( + ProcedureContext procedureContext, + String targetTableId, + String targetAlias, + String sourceSqls, + String sourceTable, + String mergeCondition, + String matchedDeleteCondition) { + return call( + procedureContext, + targetTableId, + targetAlias, + sourceSqls, + sourceTable, + mergeCondition, + "", + "", + "", + "", + matchedDeleteCondition); + } + + public String[] call( + ProcedureContext procedureContext, + String targetTableId, + String targetAlias, + String sourceSqls, + String sourceTable, + String mergeCondition, + String matchedUpsertCondition, + String matchedUpsertSetting, + String notMatchedInsertCondition, + String notMatchedInsertValues, + String matchedDeleteCondition) { + String warehouse = catalog.warehouse(); + Map catalogOptions = catalog.options(); + Identifier identifier = Identifier.fromString(targetTableId); + MergeIntoAction action = + new MergeIntoAction( + warehouse, + identifier.getDatabaseName(), + identifier.getObjectName(), + catalogOptions); + action.withTargetAlias(nullable(targetAlias)); + + if (!sourceSqls.isEmpty()) { + action.withSourceSqls(sourceSqls.split(";")); + } + + checkArgument(!sourceTable.isEmpty(), "Must specify source table."); + action.withSourceTable(sourceTable); + + checkArgument(!mergeCondition.isEmpty(), "Must specify merge condition."); + action.withMergeCondition(mergeCondition); + + if (!matchedUpsertCondition.isEmpty() || !matchedUpsertSetting.isEmpty()) { + String condition = nullable(matchedUpsertCondition); + String setting = nullable(matchedUpsertSetting); + checkNotNull(setting, "matched-upsert must set the 'matchedUpsertSetting' argument"); + action.withMatchedUpsert(condition, setting); + } + + if (!notMatchedInsertCondition.isEmpty() || !notMatchedInsertValues.isEmpty()) { + String condition = nullable(notMatchedInsertCondition); + String values = nullable(notMatchedInsertValues); + checkNotNull( + values, "not-matched-insert must set the 'notMatchedInsertValues' argument"); + action.withNotMatchedInsert(condition, values); + } + + if (!matchedDeleteCondition.isEmpty()) { + action.withMatchedDelete(matchedDeleteCondition); + } + + action.withStreamExecutionEnvironment(procedureContext.getExecutionEnvironment()); + action.validate(); + + DataStream dataStream = action.buildDataStream(); + TableResult tableResult = action.batchSink(dataStream); + JobClient jobClient = tableResult.getJobClient().get(); + + return execute(procedureContext, jobClient); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateDatabaseProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateDatabaseProcedure.java new file mode 100644 index 0000000000000..128875a8b8627 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateDatabaseProcedure.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.flink.utils.TableMigrationUtils; +import org.apache.paimon.migrate.Migrator; +import org.apache.paimon.utils.ParameterUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.util.List; + +/** Migrate procedure to migrate all hive tables in database to paimon table. */ +public class MigrateDatabaseProcedure extends ProcedureBase { + + @Override + public String identifier() { + return "migrate_database"; + } + + public String[] call( + ProcedureContext procedureContext, String connector, String sourceDatabasePath) + throws Exception { + return call(procedureContext, connector, sourceDatabasePath, ""); + } + + public String[] call( + ProcedureContext procedureContext, + String connector, + String sourceDatabasePath, + String properties) + throws Exception { + List migrators = + TableMigrationUtils.getImporters( + connector, + catalog, + sourceDatabasePath, + ParameterUtils.parseCommaSeparatedKeyValues(properties)); + + for (Migrator migrator : migrators) { + migrator.executeMigrate(); + migrator.renameTable(false); + } + + return new String[] {"Success"}; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java new file mode 100644 index 0000000000000..110b4e25fc003 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.flink.utils.TableMigrationUtils; +import org.apache.paimon.migrate.Migrator; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.util.Collections; + +/** Add file procedure to add file from hive to paimon. */ +public class MigrateFileProcedure extends ProcedureBase { + + @Override + public String identifier() { + return "migrate_file"; + } + + public String[] call( + ProcedureContext procedureContext, + String connector, + String sourceTablePath, + String targetPaimonTablePath) + throws Exception { + call(procedureContext, connector, sourceTablePath, targetPaimonTablePath, true); + return new String[] {"Success"}; + } + + public String[] call( + ProcedureContext procedureContext, + String connector, + String sourceTablePath, + String targetPaimonTablePath, + boolean deleteOrigin) + throws Exception { + migrateHandle(connector, sourceTablePath, targetPaimonTablePath, deleteOrigin); + return new String[] {"Success"}; + } + + public void migrateHandle( + String connector, + String sourceTablePath, + String targetPaimonTablePath, + boolean deleteOrigin) + throws Exception { + Identifier sourceTableId = Identifier.fromString(sourceTablePath); + Identifier targetTableId = Identifier.fromString(targetPaimonTablePath); + + if (!(catalog.tableExists(targetTableId))) { + throw new IllegalArgumentException( + "Target paimon table does not exist: " + targetPaimonTablePath); + } + + Migrator importer = + TableMigrationUtils.getImporter( + connector, + catalog, + sourceTableId.getDatabaseName(), + sourceTableId.getObjectName(), + targetTableId.getDatabaseName(), + targetTableId.getObjectName(), + Collections.emptyMap()); + importer.deleteOriginTable(deleteOrigin); + importer.executeMigrate(); + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateTableProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateTableProcedure.java new file mode 100644 index 0000000000000..39e6092d84960 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateTableProcedure.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.flink.utils.TableMigrationUtils; +import org.apache.paimon.utils.ParameterUtils; + +import org.apache.flink.table.procedure.ProcedureContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Migrate procedure to migrate hive table to paimon table. */ +public class MigrateTableProcedure extends ProcedureBase { + + private static final Logger LOG = LoggerFactory.getLogger(MigrateTableProcedure.class); + + private static final String PAIMON_SUFFIX = "_paimon_"; + + @Override + public String identifier() { + return "migrate_table"; + } + + public String[] call( + ProcedureContext procedureContext, String connector, String sourceTablePath) + throws Exception { + return call(procedureContext, connector, sourceTablePath, ""); + } + + public String[] call( + ProcedureContext procedureContext, + String connector, + String sourceTablePath, + String properties) + throws Exception { + String targetPaimonTablePath = sourceTablePath + PAIMON_SUFFIX; + + Identifier sourceTableId = Identifier.fromString(sourceTablePath); + Identifier targetTableId = Identifier.fromString(targetPaimonTablePath); + + TableMigrationUtils.getImporter( + connector, + catalog, + sourceTableId.getDatabaseName(), + sourceTableId.getObjectName(), + targetTableId.getDatabaseName(), + targetTableId.getObjectName(), + ParameterUtils.parseCommaSeparatedKeyValues(properties)) + .executeMigrate(); + + LOG.info("Last step: rename " + targetTableId + " to " + sourceTableId); + catalog.renameTable(targetTableId, sourceTableId, false); + return new String[] {"Success"}; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java new file mode 100644 index 0000000000000..d43056f9779e4 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.operation.OrphanFilesClean; +import org.apache.paimon.utils.StringUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.util.List; + +import static org.apache.paimon.operation.OrphanFilesClean.executeOrphanFilesClean; + +/** + * Remove orphan files procedure. Usage: + * + *


+ *  -- use the default file delete interval
+ *  CALL sys.remove_orphan_files('tableId')
+ *
+ *  -- use custom file delete interval
+ *  CALL sys.remove_orphan_files('tableId', '2023-12-31 23:59:59')
+ *
+ *  -- remove all tables' orphan files in db
+ *  CALL sys.remove_orphan_files('databaseName.*', '2023-12-31 23:59:59')
+ * 
+ */ +public class RemoveOrphanFilesProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "remove_orphan_files"; + + public String[] call(ProcedureContext procedureContext, String tableId) throws Exception { + return call(procedureContext, tableId, ""); + } + + public String[] call(ProcedureContext procedureContext, String tableId, String olderThan) + throws Exception { + return call(procedureContext, tableId, olderThan, false); + } + + public String[] call( + ProcedureContext procedureContext, String tableId, String olderThan, boolean dryRun) + throws Exception { + Identifier identifier = Identifier.fromString(tableId); + String databaseName = identifier.getDatabaseName(); + String tableName = identifier.getObjectName(); + + List tableCleans = + OrphanFilesClean.createOrphanFilesCleans(catalog, databaseName, tableName); + + if (!StringUtils.isBlank(olderThan)) { + tableCleans.forEach(clean -> clean.olderThan(olderThan)); + } + + if (dryRun) { + tableCleans.forEach(clean -> clean.fileCleaner(path -> {})); + } + + return executeOrphanFilesClean(tableCleans); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java new file mode 100644 index 0000000000000..0355d6dc1cab5 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.consumer.Consumer; +import org.apache.paimon.consumer.ConsumerManager; +import org.apache.paimon.table.FileStoreTable; + +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Reset consumer procedure. Usage: + * + *

+ *  -- reset the new next snapshot id in the consumer
+ *  CALL sys.reset_consumer('tableId', 'consumerId', nextSnapshotId)
+ *
+ *  -- delete consumer
+ *  CALL sys.reset_consumer('tableId', 'consumerId')
+ * 
+ */ +public class ResetConsumerProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "reset_consumer"; + + public String[] call( + ProcedureContext procedureContext, + String tableId, + String consumerId, + long nextSnapshotId) + throws Catalog.TableNotExistException { + FileStoreTable fileStoreTable = + (FileStoreTable) catalog.getTable(Identifier.fromString(tableId)); + ConsumerManager consumerManager = + new ConsumerManager( + fileStoreTable.fileIO(), + fileStoreTable.location(), + fileStoreTable.snapshotManager().branch()); + consumerManager.resetConsumer(consumerId, new Consumer(nextSnapshotId)); + + return new String[] {"Success"}; + } + + public String[] call(ProcedureContext procedureContext, String tableId, String consumerId) + throws Catalog.TableNotExistException { + FileStoreTable fileStoreTable = + (FileStoreTable) catalog.getTable(Identifier.fromString(tableId)); + ConsumerManager consumerManager = + new ConsumerManager( + fileStoreTable.fileIO(), + fileStoreTable.location(), + fileStoreTable.snapshotManager().branch()); + consumerManager.deleteConsumer(consumerId); + + return new String[] {"Success"}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RewriteFileIndexProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RewriteFileIndexProcedure.java new file mode 100644 index 0000000000000..29ae1b25b57a1 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RewriteFileIndexProcedure.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.flink.sink.NoneCopyVersionedSerializerTypeSerializerProxy; +import org.apache.paimon.flink.sink.RewriteFileIndexSink; +import org.apache.paimon.flink.source.RewriteFileIndexSource; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestEntrySerializer; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.StringUtils; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.typeutils.GenericTypeInfo; +import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.procedure.ProcedureContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.utils.ParameterUtils.getPartitions; + +/** Rewrite file index procedure to re-generated all file index. */ +public class RewriteFileIndexProcedure extends ProcedureBase { + + @Override + public String identifier() { + return "rewrite_file_index"; + } + + public String[] call(ProcedureContext procedureContext, String sourceTablePath) + throws Exception { + return call(procedureContext, sourceTablePath, ""); + } + + public String[] call( + ProcedureContext procedureContext, String sourceTablePath, String partitions) + throws Exception { + + StreamExecutionEnvironment env = procedureContext.getExecutionEnvironment(); + Table table = catalog.getTable(Identifier.fromString(sourceTablePath)); + + List> partitionList = + StringUtils.isBlank(partitions) ? null : getPartitions(partitions.split(";")); + + Predicate partitionPredicate; + if (partitionList != null) { + // This predicate is based on the row type of the original table, not bucket table. + // Because TableScan in BucketsTable is the same with FileStoreTable, + // and partition filter is done by scan. + partitionPredicate = + PredicateBuilder.or( + partitionList.stream() + .map( + p -> + PredicateBuilder.partition( + p, + ((FileStoreTable) table) + .schema() + .logicalPartitionType(), + CoreOptions.PARTITION_DEFAULT_NAME + .defaultValue())) + .toArray(Predicate[]::new)); + } else { + partitionPredicate = null; + } + + FileStoreTable storeTable = (FileStoreTable) table; + DataStreamSource source = + env.fromSource( + new RewriteFileIndexSource(storeTable, partitionPredicate), + WatermarkStrategy.noWatermarks(), + "index source", + new ManifestEntryTypeInfo()); + new RewriteFileIndexSink(storeTable).sinkFrom(source); + return execute(env, "Add file index for table: " + sourceTablePath); + } + + private static class ManifestEntryTypeInfo extends GenericTypeInfo { + + public ManifestEntryTypeInfo() { + super(ManifestEntry.class); + } + + @Override + public TypeSerializer createSerializer(ExecutionConfig config) { + return new NoneCopyVersionedSerializerTypeSerializerProxy<>( + () -> + new SimpleVersionedSerializer() { + private final ManifestEntrySerializer manifestEntrySerializer = + new ManifestEntrySerializer(); + + @Override + public int getVersion() { + return 0; + } + + @Override + public byte[] serialize(ManifestEntry manifestEntry) + throws IOException { + return manifestEntrySerializer.serializeToBytes(manifestEntry); + } + + @Override + public ManifestEntry deserialize(int i, byte[] bytes) + throws IOException { + return manifestEntrySerializer.deserializeFromBytes(bytes); + } + }); + } + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java new file mode 100644 index 0000000000000..1bf545004d93d --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.Table; + +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Rollback procedure. Usage: + * + *

+ *  -- rollback to a snapshot
+ *  CALL sys.rollback_to('tableId', snapshotId)
+ *
+ *  -- rollback to a tag
+ *  CALL sys.rollback_to('tableId', 'tagName')
+ * 
+ */ +public class RollbackToProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "rollback_to"; + + public String[] call(ProcedureContext procedureContext, String tableId, long snapshotId) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + table.rollbackTo(snapshotId); + + return new String[] {"Success"}; + } + + public String[] call(ProcedureContext procedureContext, String tableId, String tagName) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + table.rollbackTo(tagName); + + return new String[] {"Success"}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/privilege/GrantPrivilegeToUserProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/privilege/GrantPrivilegeToUserProcedure.java new file mode 100644 index 0000000000000..e57b364a2f5d1 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/privilege/GrantPrivilegeToUserProcedure.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure.privilege; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.privilege.PrivilegeType; + +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Procedure to grant privilege to a user. Privilege can be granted on the whole catalog, a database + * or a table. Only users with {@link PrivilegeType#ADMIN} privilege can perform this operation. + * Usage: + * + *

+ *  CALL sys.grant_privilege_to_user('username', 'privilege')
+ *  CALL sys.grant_privilege_to_user('username', 'privilege', 'database')
+ *  CALL sys.grant_privilege_to_user('username', 'privilege', 'database', 'table')
+ * 
+ */ +public class GrantPrivilegeToUserProcedure extends PrivilegeProcedureBase { + + public static final String IDENTIFIER = "grant_privilege_to_user"; + + public String[] call(ProcedureContext procedureContext, String user, String privilege) { + getPrivilegedCatalog().grantPrivilegeOnCatalog(user, PrivilegeType.valueOf(privilege)); + return new String[] { + String.format("User %s is granted with privilege %s on the catalog.", user, privilege) + }; + } + + public String[] call( + ProcedureContext procedureContext, String user, String privilege, String database) { + getPrivilegedCatalog() + .grantPrivilegeOnDatabase(user, database, PrivilegeType.valueOf(privilege)); + return new String[] { + String.format( + "User %s is granted with privilege %s on database %s.", + user, privilege, database) + }; + } + + public String[] call( + ProcedureContext procedureContext, + String user, + String privilege, + String database, + String table) { + Identifier identifier = Identifier.create(database, table); + getPrivilegedCatalog() + .grantPrivilegeOnTable(user, identifier, PrivilegeType.valueOf(privilege)); + return new String[] { + String.format( + "User %s is granted with privilege %s on table %s.", + user, privilege, identifier) + }; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/privilege/RevokePrivilegeFromUserProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/privilege/RevokePrivilegeFromUserProcedure.java new file mode 100644 index 0000000000000..5f511eaa61b98 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/privilege/RevokePrivilegeFromUserProcedure.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure.privilege; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.privilege.PrivilegeType; + +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Procedure to revoke privilege from a user. Privilege can be revoked from the whole catalog, a + * database or a table. Only users with {@link PrivilegeType#ADMIN} privilege can perform this + * operation. Usage: + * + *

+ *  CALL sys.revoke_privilege_from_user('username', 'privilege')
+ *  CALL sys.revoke_privilege_from_user('username', 'privilege', 'database')
+ *  CALL sys.revoke_privilege_from_user('username', 'privilege', 'database', 'table')
+ * 
+ */ +public class RevokePrivilegeFromUserProcedure extends PrivilegeProcedureBase { + + public static final String IDENTIFIER = "revoke_privilege_from_user"; + + public String[] call(ProcedureContext procedureContext, String user, String privilege) { + int count = + getPrivilegedCatalog() + .revokePrivilegeOnCatalog(user, PrivilegeType.valueOf(privilege)); + return new String[] { + String.format("User %s is revoked with privilege %s on the catalog.", user, privilege), + "Number of privileges revoked: " + count + }; + } + + public String[] call( + ProcedureContext procedureContext, String user, String privilege, String database) { + int count = + getPrivilegedCatalog() + .revokePrivilegeOnDatabase( + user, database, PrivilegeType.valueOf(privilege)); + return new String[] { + String.format( + "User %s is revoked with privilege %s on database %s.", + user, privilege, database), + "Number of privileges revoked: " + count + }; + } + + public String[] call( + ProcedureContext procedureContext, + String user, + String privilege, + String database, + String table) { + Identifier identifier = Identifier.create(database, table); + int count = + getPrivilegedCatalog() + .revokePrivilegeOnTable(user, identifier, PrivilegeType.valueOf(privilege)); + return new String[] { + String.format( + "User %s is revoked with privilege %s on table %s.", + user, privilege, identifier), + "Number of privileges revoked: " + count + }; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java index cd9a8b184f3e7..967eb9d65639b 100644 --- a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java +++ b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java @@ -19,22 +19,28 @@ package org.apache.paimon.flink.procedure; import org.apache.paimon.flink.CatalogITCaseBase; +import org.apache.paimon.privilege.NoPrivilegeException; import org.apache.paimon.table.FileStoreTable; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertThrows; /** Ensure that the legacy multiply overloaded CALL with positional arguments can be invoked. */ public class ProcedurePositionalArgumentsITCase extends CatalogITCaseBase { - @Test - public void testCallCompact() { + public void testCompactDatabaseAndTable() { sql( "CREATE TABLE T (" + " k INT," @@ -61,6 +67,166 @@ public void testCallCompact() { assertThatCode(() -> sql("CALL sys.compact('default.T', '', 'zorder', 'k', '','','5s')")) .message() .contains("sort compact do not support 'partition_idle_time'."); + + assertThatCode(() -> sql("CALL sys.compact_database('default')")) + .doesNotThrowAnyException(); + } + + @Test + public void testUserPrivileges() throws Exception { + sql( + String.format( + "CREATE CATALOG mycat WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '%s'\n" + + ")", + path)); + sql("USE CATALOG mycat"); + sql("CREATE DATABASE mydb"); + sql("CREATE DATABASE mydb2"); + sql( + "CREATE TABLE mydb.T1 (\n" + + " k INT,\n" + + " v INT,\n" + + " PRIMARY KEY (k) NOT ENFORCED\n" + + ")"); + sql("INSERT INTO mydb.T1 VALUES (1, 10), (2, 20), (3, 30)"); + sql("CALL sys.init_file_based_privilege('root-passwd')"); + + sql( + String.format( + "CREATE CATALOG anonymouscat WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '%s'\n" + + ")", + path)); + + sql("USE CATALOG anonymouscat"); + assertNoPrivilege(() -> sql("INSERT INTO mydb.T1 VALUES (1, 11), (2, 21)")); + assertNoPrivilege(() -> collect("SELECT * FROM mydb.T1 ORDER BY k")); + + sql( + String.format( + "CREATE CATALOG rootcat WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '%s',\n" + + " 'user' = 'root',\n" + + " 'password' = 'root-passwd'\n" + + ")", + path)); + sql("USE CATALOG rootcat"); + sql( + "CREATE TABLE mydb2.T2 (\n" + + " k INT,\n" + + " v INT,\n" + + " PRIMARY KEY (k) NOT ENFORCED\n" + + ")"); + sql("INSERT INTO mydb2.T2 VALUES (100, 1000), (200, 2000), (300, 3000)"); + sql("CALL sys.create_privileged_user('test', 'test-passwd')"); + sql("CALL sys.grant_privilege_to_user('test', 'CREATE_TABLE', 'mydb')"); + sql("CALL sys.grant_privilege_to_user('test', 'SELECT', 'mydb')"); + sql("CALL sys.grant_privilege_to_user('test', 'INSERT', 'mydb')"); + + sql( + String.format( + "CREATE CATALOG testcat WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '%s',\n" + + " 'user' = 'test',\n" + + " 'password' = 'test-passwd'\n" + + ")", + path)); + sql("USE CATALOG testcat"); + sql("INSERT INTO mydb.T1 VALUES (1, 12), (2, 22)"); + assertThat(collect("SELECT * FROM mydb.T1 ORDER BY k")) + .isEqualTo(Arrays.asList(Row.of(1, 12), Row.of(2, 22), Row.of(3, 30))); + sql("CREATE TABLE mydb.S1 ( a INT, b INT )"); + sql("INSERT INTO mydb.S1 VALUES (1, 100), (2, 200), (3, 300)"); + assertThat(collect("SELECT * FROM mydb.S1 ORDER BY a")) + .isEqualTo(Arrays.asList(Row.of(1, 100), Row.of(2, 200), Row.of(3, 300))); + assertNoPrivilege(() -> sql("DROP TABLE mydb.T1")); + assertNoPrivilege(() -> sql("ALTER TABLE mydb.T1 RENAME TO mydb.T2")); + assertNoPrivilege(() -> sql("DROP TABLE mydb.S1")); + assertNoPrivilege(() -> sql("ALTER TABLE mydb.S1 RENAME TO mydb.S2")); + assertNoPrivilege(() -> sql("CREATE DATABASE anotherdb")); + assertNoPrivilege(() -> sql("DROP DATABASE mydb CASCADE")); + assertNoPrivilege(() -> sql("CALL sys.create_privileged_user('test2', 'test2-passwd')")); + + sql("USE CATALOG rootcat"); + sql("CALL sys.create_privileged_user('test2', 'test2-passwd')"); + sql("CALL sys.grant_privilege_to_user('test2', 'SELECT', 'mydb2')"); + sql("CALL sys.grant_privilege_to_user('test2', 'INSERT', 'mydb', 'T1')"); + sql("CALL sys.grant_privilege_to_user('test2', 'SELECT', 'mydb', 'S1')"); + sql("CALL sys.grant_privilege_to_user('test2', 'CREATE_DATABASE')"); + + sql( + String.format( + "CREATE CATALOG test2cat WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '%s',\n" + + " 'user' = 'test2',\n" + + " 'password' = 'test2-passwd'\n" + + ")", + path)); + sql("USE CATALOG test2cat"); + sql("INSERT INTO mydb.T1 VALUES (1, 13), (2, 23)"); + assertNoPrivilege(() -> collect("SELECT * FROM mydb.T1 ORDER BY k")); + assertNoPrivilege(() -> sql("CREATE TABLE mydb.S2 ( a INT, b INT )")); + assertNoPrivilege(() -> sql("INSERT INTO mydb.S1 VALUES (1, 100), (2, 200), (3, 300)")); + assertThat(collect("SELECT * FROM mydb.S1 ORDER BY a")) + .isEqualTo(Arrays.asList(Row.of(1, 100), Row.of(2, 200), Row.of(3, 300))); + assertNoPrivilege( + () -> sql("INSERT INTO mydb2.T2 VALUES (100, 1001), (200, 2001), (300, 3001)")); + assertThat(collect("SELECT * FROM mydb2.T2 ORDER BY k")) + .isEqualTo(Arrays.asList(Row.of(100, 1000), Row.of(200, 2000), Row.of(300, 3000))); + sql("CREATE DATABASE anotherdb"); + assertNoPrivilege(() -> sql("DROP TABLE mydb.T1")); + assertNoPrivilege(() -> sql("ALTER TABLE mydb.T1 RENAME TO mydb.T2")); + assertNoPrivilege(() -> sql("DROP TABLE mydb.S1")); + assertNoPrivilege(() -> sql("ALTER TABLE mydb.S1 RENAME TO mydb.S2")); + assertNoPrivilege(() -> sql("DROP DATABASE mydb CASCADE")); + assertNoPrivilege(() -> sql("CALL sys.create_privileged_user('test3', 'test3-passwd')")); + + sql("USE CATALOG rootcat"); + assertThat(collect("SELECT * FROM mydb.T1 ORDER BY k")) + .isEqualTo(Arrays.asList(Row.of(1, 13), Row.of(2, 23), Row.of(3, 30))); + sql("CALL sys.revoke_privilege_from_user('test2', 'SELECT')"); + sql("CALL sys.drop_privileged_user('test')"); + + sql("USE CATALOG testcat"); + Exception e = + assertThrows(Exception.class, () -> collect("SELECT * FROM mydb.T1 ORDER BY k")); + assertThat(e).hasRootCauseMessage("User test not found, or password incorrect."); + + sql("USE CATALOG test2cat"); + assertNoPrivilege(() -> collect("SELECT * FROM mydb.S1 ORDER BY a")); + assertNoPrivilege(() -> collect("SELECT * FROM mydb2.T2 ORDER BY k")); + sql("INSERT INTO mydb.T1 VALUES (1, 14), (2, 24)"); + + sql("USE CATALOG rootcat"); + assertThat(collect("SELECT * FROM mydb.T1 ORDER BY k")) + .isEqualTo(Arrays.asList(Row.of(1, 14), Row.of(2, 24), Row.of(3, 30))); + sql("DROP DATABASE mydb CASCADE"); + sql("DROP DATABASE mydb2 CASCADE"); + } + + private List collect(String sql) throws Exception { + List result = new ArrayList<>(); + try (CloseableIterator it = tEnv.executeSql(sql).collect()) { + while (it.hasNext()) { + result.add(it.next()); + } + } + return result; + } + + private void assertNoPrivilege(Executable executable) { + Exception e = assertThrows(Exception.class, executable); + if (e.getCause() != null) { + assertThat(e).hasRootCauseInstanceOf(NoPrivilegeException.class); + } else { + assertThat(e).isInstanceOf(NoPrivilegeException.class); + } } @Test @@ -88,4 +254,235 @@ private List read(FileStoreTable table) throws IOException { .forEachRemaining(row -> ret.add(row.getString(0) + ":" + row.getString(1))); return ret; } + + @Test + public void testCreateDeleteTag() { + sql( + "CREATE TABLE T (" + + " k STRING," + + " dt STRING," + + " PRIMARY KEY (k, dt) NOT ENFORCED" + + ") PARTITIONED BY (dt) WITH (" + + " 'bucket' = '1'" + + ")"); + sql("insert into T values('k', '2024-01-01')"); + sql("insert into T values('k2', '2024-01-02')"); + + sql("CALL sys.create_tag('default.T', 'tag1')"); + + assertThat( + sql("select * from T /*+ OPTIONS('scan.tag-name'='tag1') */").stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("+I[k2, 2024-01-02]", "+I[k, 2024-01-01]"); + + sql("CALL sys.rollback_to('default.T', 'tag1')"); + + assertThat(sql("select * from T").stream().map(Row::toString)) + .containsExactlyInAnyOrder("+I[k2, 2024-01-02]", "+I[k, 2024-01-01]"); + + sql("CALL sys.delete_tag('default.T', 'tag1')"); + + assertThatThrownBy( + () -> + sql("select * from T /*+ OPTIONS('scan.tag-name'='tag1') */") + .stream() + .map(Row::toString)) + .hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Tag 'tag1' doesn't exist."); + } + + @Test + public void testCreateDeleteAndForwardBranch() throws Exception { + sql( + "CREATE TABLE T (" + + " pt INT" + + ", k INT" + + ", v STRING" + + ", PRIMARY KEY (pt, k) NOT ENFORCED" + + " ) PARTITIONED BY (pt) WITH (" + + " 'bucket' = '2'" + + " )"); + + // snapshot 1. + sql("INSERT INTO T VALUES(1, 10, 'apple')"); + + // snapshot 2. + sql("INSERT INTO T VALUES(1, 20, 'dog')"); + + sql("CALL sys.create_tag('default.T', 'tag1', 1)"); + + sql("CALL sys.create_tag('default.T', 'tag2', 2)"); + + sql("CALL sys.create_branch('default.T', 'test', 'tag1')"); + sql("CALL sys.create_branch('default.T', 'test2', 'tag2')"); + + assertThat(collectToString("SELECT branch_name, created_from_snapshot FROM `T$branches`")) + .containsExactlyInAnyOrder("+I[test, 1]", "+I[test2, 2]"); + + sql("CALL sys.delete_branch('default.T', 'test')"); + + assertThat(collectToString("SELECT branch_name, created_from_snapshot FROM `T$branches`")) + .containsExactlyInAnyOrder("+I[test2, 2]"); + + sql("CALL sys.fast_forward('default.T', 'test2')"); + + // Branch `test` replaces the main branch. + assertThat(collectToString("SELECT * FROM `T`")) + .containsExactlyInAnyOrder("+I[1, 10, apple]", "+I[1, 20, dog]"); + } + + private List collectToString(String sql) throws Exception { + List result = new ArrayList<>(); + try (CloseableIterator it = tEnv.executeSql(sql).collect()) { + while (it.hasNext()) { + result.add(it.next().toString()); + } + } + return result; + } + + @Test + public void testPartitionMarkDone() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + + assertThatCode(() -> sql("CALL sys.mark_partition_done('default.T', 'pt = 0')")) + .doesNotThrowAnyException(); + } + + @Test + public void testMergeInto() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + sql( + "CREATE TABLE S (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + + assertThatCode( + () -> + sql( + "CALL sys.merge_into('default.T', 'TT', '', 'S', 'TT.k = S.k', '', '', '', '', 'S.v IS NULL')")) + .doesNotThrowAnyException(); + } + + @Test + public void testMigrateProcedures() { + sql( + "CREATE TABLE S (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + + assertThatThrownBy(() -> sql("CALL sys.migrate_database('hive', 'default', '')")) + .hasMessageContaining("Only support Hive Catalog."); + assertThatThrownBy(() -> sql("CALL sys.migrate_table('hive', 'default.T', '')")) + .hasMessageContaining("Only support Hive Catalog."); + assertThatThrownBy(() -> sql("CALL sys.migrate_file('hive', 'default.T', 'default.S')")) + .hasMessageContaining("Only support Hive Catalog."); + } + + @Test + public void testQueryService() { + sql( + "CREATE TABLE DIM (k INT PRIMARY KEY NOT ENFORCED, v INT) WITH ('bucket' = '2', 'continuous.discovery-interval' = '1ms')"); + assertThatCode( + () -> { + CloseableIterator service = + streamSqlIter("CALL sys.query_service('default.DIM', 2)"); + service.close(); + }) + .doesNotThrowAnyException(); + } + + @Test + public void testRemoveOrphanFiles() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + assertThatCode(() -> sql("CALL sys.remove_orphan_files('default.T')")) + .doesNotThrowAnyException(); + } + + @Test + public void testRepair() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + assertThatThrownBy(() -> sql("CALL sys.repair('default.T')")) + .hasStackTraceContaining("Catalog.repairTable"); + } + + @Test + public void testResetConsumer() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + assertThatCode(() -> sql("CALL sys.reset_consumer('default.T', 'myid')")) + .doesNotThrowAnyException(); + } + + @Test + public void testRewriteFileIndex() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " pt INT," + + " PRIMARY KEY (k, pt) NOT ENFORCED" + + ") PARTITIONED BY (pt) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + assertThatCode(() -> sql("CALL sys.rewrite_file_index('default.T', 'pt = 0')")) + .doesNotThrowAnyException(); + } }