From 2e1371365b2c207da16286350845bcef86e5f0a1 Mon Sep 17 00:00:00 2001 From: saihemanth Date: Thu, 19 Dec 2024 18:03:13 +0530 Subject: [PATCH 1/6] HIVE-28668: Hive should emit fewer events for truncate table operation --- .../listener/TestDbNotificationListener.java | 43 ++++++ .../hadoop/hive/metastore/AlterHandler.java | 8 +- .../hadoop/hive/metastore/HMSHandler.java | 123 ++++++------------ .../hive/metastore/HiveAlterHandler.java | 20 ++- 4 files changed, 100 insertions(+), 94 deletions(-) diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index dec716089c5b..50d40cded687 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -857,6 +857,49 @@ public void alterPartitions() throws Exception { testEventCounts(defaultDbName, firstEventId, null, null, 3); } + @Test + public void testTruncatePartitionedTable() throws Exception { + String defaultDbName = "default"; + String unPartitionedTblName = "unPartitionedTable"; + new TableBuilder() + .setDbName(defaultDbName) + .setTableName(unPartitionedTblName) + .addCol("col1", "int") + .setLocation(testTempDir) + .create(msClient, new HiveConf()); + + Table table = msClient.getTable(new GetTableRequest(defaultDbName, + unPartitionedTblName)); + msClient.truncateTable(defaultDbName, unPartitionedTblName, null); + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(2, rsp.getEventsSize()); // create unpartitioned table + alter table events + + String partitionedTblName = "partitionedTbl"; + new TableBuilder() + .setDbName(defaultDbName) + .setTableName(partitionedTblName) + .addCol("col1", "int") + .addPartCol("col2", "int") + .addPartCol("col3", "string") + .setLocation(testTempDir) + .create(msClient, new HiveConf()); + table = msClient.getTable(new GetTableRequest(defaultDbName, + partitionedTblName)); + List partitions = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List values = Arrays.asList(i + "", "part" + i); + Partition part = new Partition(values, defaultDbName, partitionedTblName, + 0, 0, table.getSd(), emptyParameters); + partitions.add(part); + } + msClient.add_partitions(partitions); + msClient.truncateTable(defaultDbName, partitionedTblName, null); + rsp = msClient.getNextNotification(firstEventId, 0, null); + // 5 events - create unpartitioned table, alter table events + // create partitioned table, add partition, alter table events. + assertEquals(5, rsp.getEventsSize()); + } + @Test public void dropPartition() throws Exception { String defaultDbName = "default"; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index 6edbef393971..cd6a2566620e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@ -62,7 +62,7 @@ public interface AlterHandler extends Configurable { default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, String name, Table newTable, EnvironmentContext envContext) throws InvalidOperationException, MetaException { - alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null, null); + alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null, null, false); } /** @@ -82,6 +82,8 @@ default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbna * new table object * @param handler * HMSHandle object (required to log event notification) + * @param writeIdList write id list for the table + * @param isTruncateOp boolean flag to specify if this is truncate operation * @throws InvalidOperationException * thrown if the newTable object is invalid * @throws MetaException @@ -89,7 +91,7 @@ default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbna */ void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, String name, Table newTable, EnvironmentContext envContext, - IHMSHandler handler, String writeIdList) + IHMSHandler handler, String writeIdList, boolean isTruncateOp) throws InvalidOperationException, MetaException; /** @@ -200,6 +202,6 @@ List alterPartitions(final RawStore msdb, Warehouse wh, List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, String writeIdList, long writeId, - IHMSHandler handler) + IHMSHandler handler, boolean isTruncateOp) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index 2c1e53b11d67..c43a4a1ed08a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -3362,93 +3362,47 @@ private void updateStatsForTruncate(Map props, EnvironmentContext return; } - private void alterPartitionForTruncate(RawStore ms, String catName, String dbName, String tableName, - Table table, Partition partition, String validWriteIds, long writeId) throws Exception { + private void alterPartitionsForTruncate(RawStore ms, String catName, String dbName, String tableName, + Table table, List partitions, String validWriteIds, long writeId) throws Exception { EnvironmentContext environmentContext = new EnvironmentContext(); - updateStatsForTruncate(partition.getParameters(), environmentContext); - - if (!transactionalListeners.isEmpty()) { - MetaStoreListenerNotifier.notifyEvent(transactionalListeners, - EventType.ALTER_PARTITION, - new AlterPartitionEvent(partition, partition, table, true, true, - writeId, this)); - } - - if (!listeners.isEmpty()) { - MetaStoreListenerNotifier.notifyEvent(listeners, - EventType.ALTER_PARTITION, - new AlterPartitionEvent(partition, partition, table, true, true, - writeId, this)); - } - - if (writeId > 0) { - partition.setWriteId(writeId); + for (Partition partition: partitions) { + updateStatsForTruncate(partition.getParameters(), environmentContext); + if (writeId > 0) { + partition.setWriteId(writeId); + } } - alterHandler.alterPartition(ms, wh, catName, dbName, tableName, null, partition, - environmentContext, this, validWriteIds); + alterHandler.alterPartitions(ms, wh, catName, dbName, tableName, partitions, environmentContext, + validWriteIds, writeId, this, true); } private void alterTableStatsForTruncate(RawStore ms, String catName, String dbName, - String tableName, Table table, List partNames, - String validWriteIds, long writeId) throws Exception { - if (partNames == null) { - if (0 != table.getPartitionKeysSize()) { - for (Partition partition : ms.getPartitions(catName, dbName, tableName, -1)) { - alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition, - validWriteIds, writeId); - } - } else { - EnvironmentContext environmentContext = new EnvironmentContext(); - updateStatsForTruncate(table.getParameters(), environmentContext); - - boolean isReplicated = isDbReplicationTarget(ms.getDatabase(catName, dbName)); - if (!transactionalListeners.isEmpty()) { - MetaStoreListenerNotifier.notifyEvent(transactionalListeners, - EventType.ALTER_TABLE, - new AlterTableEvent(table, table, true, true, - writeId, this, isReplicated)); - } - - if (!listeners.isEmpty()) { - MetaStoreListenerNotifier.notifyEvent(listeners, - EventType.ALTER_TABLE, - new AlterTableEvent(table, table, true, true, - writeId, this, isReplicated)); - } - - // TODO: this should actually pass thru and set writeId for txn stats. - if (writeId > 0) { - table.setWriteId(writeId); - } - alterHandler.alterTable(ms, wh, catName, dbName, tableName, table, - environmentContext, this, validWriteIds); - } + String tableName, Table table, List partitionsList, + String validWriteIds, long writeId) throws Exception { + if (!partitionsList.isEmpty()) { + alterPartitionsForTruncate(ms, catName, dbName, tableName, table, partitionsList, + validWriteIds, writeId); } else { - for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { - alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition, - validWriteIds, writeId); + EnvironmentContext environmentContext = new EnvironmentContext(); + updateStatsForTruncate(table.getParameters(), environmentContext); + + // TODO: this should actually pass thru and set writeId for txn stats. + if (writeId > 0) { + table.setWriteId(writeId); } + alterHandler.alterTable(ms, wh, catName, dbName, tableName, table, + environmentContext, this, validWriteIds, true); } return; } - private List getLocationsForTruncate(final RawStore ms, - final String catName, - final String dbName, - final String tableName, - final Table table, - final List partNames) throws Exception { + private List getLocationsForTruncate(final RawStore ms, final String catName, + final String dbName, final String tableName, final Table table, + List partitionsList) throws Exception { List locations = new ArrayList<>(); - if (partNames == null) { - if (0 != table.getPartitionKeysSize()) { - for (Partition partition : ms.getPartitions(catName, dbName, tableName, -1)) { - locations.add(new Path(partition.getSd().getLocation())); - } - } else { - locations.add(new Path(table.getSd().getLocation())); - } + if (partitionsList.isEmpty()) { + locations.add(new Path(table.getSd().getLocation())); } else { - for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { + for (Partition partition : partitionsList) { locations.add(new Path(partition.getSd().getLocation())); } } @@ -3490,7 +3444,16 @@ private void truncateTableInternal(String dbName, String tableName, List .map(prop -> prop.get(TRUNCATE_SKIP_DATA_DELETION)) .map(Boolean::parseBoolean) .orElse(false); - + List partitionsList = new ArrayList<>(); + if (partNames == null) { + if (0 != tbl.getPartitionKeysSize()) { + partitionsList = getMS().getPartitions(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], + tableName, -1); + } + } else { + partitionsList = getMS().getPartitionsByNames(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], + tableName, partNames); + } if (TxnUtils.isTransactionalTable(tbl) || !skipDataDeletion) { if (!skipDataDeletion) { isSkipTrash = MetaStoreUtils.isSkipTrash(tbl.getParameters()); @@ -3500,7 +3463,7 @@ private void truncateTableInternal(String dbName, String tableName, List } // This is not transactional for (Path location : getLocationsForTruncate(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, - tbl, partNames)) { + tbl, partitionsList)) { if (!skipDataDeletion) { truncateDataFiles(location, isSkipTrash, needCmRecycle); } else { @@ -3513,7 +3476,7 @@ private void truncateTableInternal(String dbName, String tableName, List // Alter the table/partition stats and also notify truncate table event alterTableStatsForTruncate(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], - tableName, tbl, partNames, validWriteIds, writeId); + tableName, tbl, partitionsList, validWriteIds, writeId); } catch (Exception e) { throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class) .convertIfInstance(IOException.class, MetaException.class) @@ -6141,8 +6104,8 @@ private void alter_partitions_with_environment_context(String catName, String db } firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table, null, tmpPart, this)); } - oldParts = alterHandler.alterPartitions(getMS(), wh, - catName, db_name, tbl_name, new_parts, environmentContext, writeIdList, writeId, this); + oldParts = alterHandler.alterPartitions(getMS(), wh, catName, db_name, tbl_name, new_parts, + environmentContext, writeIdList, writeId, this, false); Iterator olditr = oldParts.iterator(); for (Partition tmpPart : new_parts) { @@ -6294,7 +6257,7 @@ private void alter_table_core(String catName, String dbname, String name, Table } firePreEvent(new PreAlterTableEvent(oldt, newTable, this)); alterHandler.alterTable(getMS(), wh, catName, dbname, name, newTable, - envContext, this, validWriteIdList); + envContext, this, validWriteIdList, false); success = true; } catch (Exception e) { ex = e; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index fbbbca8cb498..c824cd6f5edf 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -101,7 +101,7 @@ public void setConf(Configuration conf) { @Override public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, String name, Table newt, EnvironmentContext environmentContext, - IHMSHandler handler, String writeIdList) + IHMSHandler handler, String writeIdList, boolean isTruncateOp) throws InvalidOperationException, MetaException { catName = normalizeIdentifier(catName); name = normalizeIdentifier(name); @@ -448,7 +448,7 @@ public List run(List input) throws Exception { if (transactionalListeners != null && !transactionalListeners.isEmpty()) { txnAlterTableEventResponses = MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_TABLE, - new AlterTableEvent(oldt, newt, false, true, + new AlterTableEvent(oldt, newt, isTruncateOp, true, newt.getWriteId(), handler, isReplicated), environmentContext); } @@ -510,7 +510,7 @@ public List run(List input) throws Exception { // make this call whether the event failed or succeeded. To make this behavior consistent, // this call is made for failed events also. MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ALTER_TABLE, - new AlterTableEvent(oldt, newt, false, success, newt.getWriteId(), handler, isReplicated), + new AlterTableEvent(oldt, newt, isTruncateOp, success, newt.getWriteId(), handler, isReplicated), environmentContext, txnAlterTableEventResponses, msdb); } } @@ -800,7 +800,7 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final EnvironmentContext environmentContext) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { return alterPartitions(msdb, wh, MetaStoreUtils.getDefaultCatalog(conf), dbname, name, new_parts, - environmentContext, null, -1, null); + environmentContext, null, -1, null, false); } private Map, Partition> getExistingPartitions(final RawStore msdb, @@ -826,11 +826,9 @@ private Map, Partition> getExistingPartitions(final RawStore msdb, @Override public List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, - final String dbname, final String name, - final List new_parts, - EnvironmentContext environmentContext, - String writeIdList, long writeId, - IHMSHandler handler) + final String dbname, final String name, final List new_parts, + EnvironmentContext environmentContext, String writeIdList, long writeId, + IHMSHandler handler, boolean isTruncateOp) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { List oldParts = new ArrayList<>(); List> partValsList = new ArrayList<>(); @@ -891,12 +889,12 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final MetastoreConf.ConfVars.NOTIFICATION_ALTER_PARTITIONS_V2_ENABLED); if (shouldSendSingleEvent) { MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITIONS, - new AlterPartitionsEvent(oldParts, new_parts, tbl, false, true, handler), environmentContext); + new AlterPartitionsEvent(oldParts, new_parts, tbl, isTruncateOp, true, handler), environmentContext); } else { for (Partition newPart : new_parts) { Partition oldPart = oldPartMap.get(newPart.getValues()); MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITION, - new AlterPartitionEvent(oldPart, newPart, tbl, false, true, newPart.getWriteId(), handler), + new AlterPartitionEvent(oldPart, newPart, tbl, isTruncateOp, true, newPart.getWriteId(), handler), environmentContext); } } From b836db50aacd05546c011638aecf993dfcb04667 Mon Sep 17 00:00:00 2001 From: saihemanth Date: Tue, 14 Jan 2025 23:09:24 +0530 Subject: [PATCH 2/6] Address test failures --- .../message/TruncatePartitionHandler.java | 55 ++++++++++---- .../hadoop/hive/metastore/AlterHandler.java | 72 ++++++++++--------- .../hadoop/hive/metastore/HMSHandler.java | 11 +-- 3 files changed, 88 insertions(+), 50 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java index 2af9f1354a2e..de19e24220bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java @@ -19,7 +19,10 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.AlterPartitionsMessage; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.table.misc.truncate.TruncateTableDesc; import org.apache.hadoop.hive.ql.exec.Task; @@ -27,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -35,17 +39,39 @@ public class TruncatePartitionHandler extends AbstractMessageHandler { @Override public List> handle(Context context) throws SemanticException { - AlterPartitionMessage msg = deserializer.getAlterPartitionMessage(context.dmd.getPayload()); - final TableName tName = TableName.fromString(msg.getTable(), null, - context.isDbNameEmpty() ? msg.getDB() : context.dbName); - - Map partSpec = new LinkedHashMap<>(); + final TableName tName; org.apache.hadoop.hive.metastore.api.Table tblObj; try { - tblObj = msg.getTableObj(); - Iterator afterIterator = msg.getPtnObjAfter().getValuesIterator(); - for (FieldSchema fs : tblObj.getPartitionKeys()) { - partSpec.put(fs.getName(), afterIterator.next()); + if (MetastoreConf.getBoolVar(context.hiveConf, + MetastoreConf.ConfVars.NOTIFICATION_ALTER_PARTITIONS_V2_ENABLED)) { + AlterPartitionsMessage singleMsg = deserializer.getAlterPartitionsMessage( + context.dmd.getPayload()); + tName = TableName.fromString(singleMsg.getTable(), null, + context.isDbNameEmpty() ? singleMsg.getDB() : context.dbName); + tblObj = singleMsg.getTableObj(); + List> afterPartitionsList = singleMsg.getPartitions(); + List> childTaskList = new ArrayList<>(); + for(Map afterIteratorMap : afterPartitionsList) { + Iterator afterIterator = afterIteratorMap.values().iterator(); + Map partSpec = new LinkedHashMap<>(); + for (FieldSchema fs : tblObj.getPartitionKeys()) { + partSpec.put(fs.getName(), afterIterator.next()); + } + childTaskList.addAll(handleSingleAlterPartition(context, tName, partSpec, + singleMsg.getWriteId())); + } + return childTaskList; + } else { + AlterPartitionMessage msg = deserializer.getAlterPartitionMessage(context.dmd.getPayload()); + tName = TableName.fromString(msg.getTable(), null, + context.isDbNameEmpty() ? msg.getDB() : context.dbName); + tblObj = msg.getTableObj(); + Iterator afterIterator = msg.getPtnObjAfter().getValuesIterator(); + Map partSpec = new LinkedHashMap<>(); + for (FieldSchema fs : tblObj.getPartitionKeys()) { + partSpec.put(fs.getName(), afterIterator.next()); + } + return handleSingleAlterPartition(context, tName, partSpec, msg.getWriteId()); } } catch (Exception e) { if (!(e instanceof SemanticException)) { @@ -54,18 +80,19 @@ public List> handle(Context context) throws SemanticException { throw (SemanticException) e; } } + } + private List> handleSingleAlterPartition(Context context, TableName tName, + Map partSpec, Long writeId) throws SemanticException { TruncateTableDesc truncateTableDesc = new TruncateTableDesc( - tName, partSpec, - context.eventOnlyReplicationSpec()); - truncateTableDesc.setWriteId(msg.getWriteId()); + tName, partSpec, context.eventOnlyReplicationSpec()); + truncateTableDesc.setWriteId(writeId); Task truncatePtnTask = TaskFactory.get( new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc, true, - context.getDumpDirectory(), context.getMetricCollector()), context.hiveConf); + context.getDumpDirectory(), context.getMetricCollector()), context.hiveConf); context.log.debug("Added truncate ptn task : {}:{}:{}", truncatePtnTask.getId(), truncateTableDesc.getTableName(), truncateTableDesc.getWriteId()); updatedMetadata.set(context.dmd.getEventTo().toString(), tName.getDb(), tName.getTable(), partSpec); - try { return ReplUtils.addChildTask(truncatePtnTask); } catch (Exception e) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index cd6a2566620e..ed87a33427a8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@ -36,7 +36,7 @@ public interface AlterHandler extends Configurable { /** * @deprecated As of release 2.2.0. Replaced by {@link #alterTable(RawStore, Warehouse, String, - * String, String, Table, EnvironmentContext, IHMSHandler, String)} + * String, String, Table, EnvironmentContext, IHMSHandler, String, boolean)} * * handles alter table, the changes could be cascaded to partitions if applicable * @@ -53,6 +53,8 @@ public interface AlterHandler extends Configurable { * newTable.tableName if alter op is not a rename. * @param newTable * new table object + * @param envContext + * environment context variable * @throws InvalidOperationException * thrown if the newTable object is invalid * @throws MetaException @@ -84,6 +86,7 @@ default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbna * HMSHandle object (required to log event notification) * @param writeIdList write id list for the table * @param isTruncateOp boolean flag to specify if this is truncate operation + * @param envContext environment context variable * @throws InvalidOperationException * thrown if the newTable object is invalid * @throws MetaException @@ -96,13 +99,14 @@ void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, /** * @deprecated As of release 2.2.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, - * String, String, List, EnvironmentContext, String, long, IHMSHandler)} + * String, String, List, EnvironmentContext, String, long, IHMSHandler, boolean)} * * handles alter partition * * @param msdb * object to get metadata * @param wh + * physical warehouse class * @param dbname * database of the partition being altered * @param name @@ -112,10 +116,11 @@ void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, * @param new_part * new partition object * @return the altered partition - * @throws InvalidOperationException - * @throws InvalidObjectException - * @throws AlreadyExistsException - * @throws MetaException + * @throws InvalidOperationException thrown if the operation is invalid + * @throws InvalidObjectException thrown if the new_part object is invalid + * @throws AlreadyExistsException thrown if the new_part object already exists + * @throws MetaException thrown if there is any other error + * @throws NoSuchObjectException thrown if there is no such object */ @Deprecated Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, @@ -138,13 +143,16 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, * original values of the partition being altered * @param new_part * new partition object + * @param environmentContext environment context variable * @param handler * HMSHandle object (required to log event notification) + * @param validWriteIds valid write id list for the table * @return the altered partition - * @throws InvalidOperationException - * @throws InvalidObjectException - * @throws AlreadyExistsException - * @throws MetaException + * @throws InvalidOperationException thrown if the operation is invalid + * @throws InvalidObjectException thrown if the new_part object is invalid + * @throws AlreadyExistsException thrown if the new_part object already exists + * @throws MetaException thrown if there is any other error + * @throws NoSuchObjectException thrown if there is no such object */ Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List part_vals, @@ -154,24 +162,25 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName /** * @deprecated As of release 3.0.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, - * String, String, List, EnvironmentContext, String, long, IHMSHandler)} + * String, String, List, EnvironmentContext, String, long, IHMSHandler, boolean)} * * handles alter partitions * * @param msdb * object to get metadata - * @param wh + * @param wh physical warehouse class * @param dbname * database of the partition being altered * @param name * table of the partition being altered * @param new_parts * new partition list + * @param environmentContext environment context variable * @return the altered partition list - * @throws InvalidOperationException - * @throws InvalidObjectException - * @throws AlreadyExistsException - * @throws MetaException + * @throws InvalidOperationException thrown if the operation is invalid + * @throws InvalidObjectException thrown if the new_parts object is invalid + * @throws AlreadyExistsException thrown if the new_part object already exists + * @throws MetaException thrown if there is any other error */ @Deprecated List alterPartitions(final RawStore msdb, Warehouse wh, @@ -181,23 +190,22 @@ List alterPartitions(final RawStore msdb, Warehouse wh, /** * handles alter partitions - * - * @param msdb - * object to get metadata - * @param wh - * @param dbname - * database of the partition being altered - * @param name - * table of the partition being altered - * @param new_parts - * new partition list - * @param handler - * HMSHandle object (required to log event notification) + * @param msdb object to get metadata + * @param wh physical warehouse class + * @param catName catalog name of the partition being altered + * @param dbname database of the partition being altered + * @param name table of the partition being altered + * @param new_parts new partition list + * @param environmentContext environment context variable + * @param writeIdList write id list for the table + * @param writeId writeId for the table + * @param handler HMSHandle object (required to log event notification) + * @param isTruncateOp whether the operation is truncate * @return the altered partition list - * @throws InvalidOperationException - * @throws InvalidObjectException - * @throws AlreadyExistsException - * @throws MetaException + * @throws InvalidOperationException thrown if the operation is invalid + * @throws InvalidObjectException thrown if the new_parts object is invalid + * @throws AlreadyExistsException thrown if the new_part object already exists + * @throws MetaException thrown if there is any other error */ List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List new_parts, diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index c43a4a1ed08a..e25dc2b48257 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -3365,6 +3365,9 @@ private void updateStatsForTruncate(Map props, EnvironmentContext private void alterPartitionsForTruncate(RawStore ms, String catName, String dbName, String tableName, Table table, List partitions, String validWriteIds, long writeId) throws Exception { EnvironmentContext environmentContext = new EnvironmentContext(); + if (partitions.isEmpty()) { + return; + } for (Partition partition: partitions) { updateStatsForTruncate(partition.getParameters(), environmentContext); if (writeId > 0) { @@ -3378,7 +3381,7 @@ private void alterPartitionsForTruncate(RawStore ms, String catName, String dbNa private void alterTableStatsForTruncate(RawStore ms, String catName, String dbName, String tableName, Table table, List partitionsList, String validWriteIds, long writeId) throws Exception { - if (!partitionsList.isEmpty()) { + if (0 != table.getPartitionKeysSize()) { alterPartitionsForTruncate(ms, catName, dbName, tableName, table, partitionsList, validWriteIds, writeId); } else { @@ -3399,12 +3402,12 @@ private List getLocationsForTruncate(final RawStore ms, final String catNa final String dbName, final String tableName, final Table table, List partitionsList) throws Exception { List locations = new ArrayList<>(); - if (partitionsList.isEmpty()) { - locations.add(new Path(table.getSd().getLocation())); - } else { + if (0 != table.getPartitionKeysSize()) { for (Partition partition : partitionsList) { locations.add(new Path(partition.getSd().getLocation())); } + } else { + locations.add(new Path(table.getSd().getLocation())); } return locations; } From bc35047960ca6df9a5d7cac1f3bbadf761551f21 Mon Sep 17 00:00:00 2001 From: saihemanth Date: Tue, 21 Jan 2025 16:14:36 +0530 Subject: [PATCH 3/6] Address review comments --- .../message/TruncatePartitionHandler.java | 6 ++---- .../gen-cpp/hive_metastore_constants.cpp | 2 ++ .../thrift/gen-cpp/hive_metastore_constants.h | 1 + .../api/hive_metastoreConstants.java | 2 ++ .../gen/thrift/gen-php/metastore/Constant.php | 6 ++++++ .../thrift/gen-py/hive_metastore/constants.py | 1 + .../thrift/gen-rb/hive_metastore_constants.rb | 2 ++ .../src/main/thrift/hive_metastore.thrift | 1 + .../hadoop/hive/metastore/AlterHandler.java | 14 ++++++------- .../hadoop/hive/metastore/HMSHandler.java | 8 +++---- .../hive/metastore/HiveAlterHandler.java | 21 ++++++++++++++++--- 11 files changed, 45 insertions(+), 19 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java index de19e24220bf..5db0ef990533 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java @@ -46,16 +46,14 @@ public List> handle(Context context) throws SemanticException { MetastoreConf.ConfVars.NOTIFICATION_ALTER_PARTITIONS_V2_ENABLED)) { AlterPartitionsMessage singleMsg = deserializer.getAlterPartitionsMessage( context.dmd.getPayload()); - tName = TableName.fromString(singleMsg.getTable(), null, - context.isDbNameEmpty() ? singleMsg.getDB() : context.dbName); tblObj = singleMsg.getTableObj(); + tName = TableName.fromString(tblObj.getTableName(), null, tblObj.getDbName()); List> afterPartitionsList = singleMsg.getPartitions(); List> childTaskList = new ArrayList<>(); for(Map afterIteratorMap : afterPartitionsList) { - Iterator afterIterator = afterIteratorMap.values().iterator(); Map partSpec = new LinkedHashMap<>(); for (FieldSchema fs : tblObj.getPartitionKeys()) { - partSpec.put(fs.getName(), afterIterator.next()); + partSpec.put(fs.getName(), afterIteratorMap.get(fs.getName())); } childTaskList.addAll(handleSingleAlterPartition(context, tName, partSpec, singleMsg.getWriteId())); diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp index 9f0b0c8cf8d4..394d4d39b1c7 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp @@ -99,6 +99,8 @@ hive_metastoreConstants::hive_metastoreConstants() { EXPECTED_PARAMETER_VALUE = "expected_parameter_value"; + IS_TRUNCATE_OP = "__is_truncate_op__"; + } }}} // namespace diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h index 504b54a01d99..12d57891d7d8 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_constants.h @@ -59,6 +59,7 @@ class hive_metastoreConstants { std::string WRITE_ID; std::string EXPECTED_PARAMETER_KEY; std::string EXPECTED_PARAMETER_VALUE; + std::string IS_TRUNCATE_OP; }; extern const hive_metastoreConstants g_hive_metastore_constants; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java index f5a102ab9647..29c4f0ade9fc 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java @@ -97,4 +97,6 @@ public static final java.lang.String EXPECTED_PARAMETER_VALUE = "expected_parameter_value"; + public static final java.lang.String IS_TRUNCATE_OP = "__is_truncate_op__"; + } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php index 84961065fd53..8d151d4b4b31 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Constant.php @@ -62,6 +62,7 @@ final class Constant extends \Thrift\Type\TConstant static protected $WRITE_ID; static protected $EXPECTED_PARAMETER_KEY; static protected $EXPECTED_PARAMETER_VALUE; + static protected $IS_TRUNCATE_OP; protected static function init_DDL_TIME() { @@ -282,4 +283,9 @@ protected static function init_EXPECTED_PARAMETER_VALUE() { return "expected_parameter_value"; } + + protected static function init_IS_TRUNCATE_OP() + { + return "__is_truncate_op__"; + } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py index b5891397a6e2..92b0835cea78 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/constants.py @@ -56,3 +56,4 @@ WRITE_ID = "writeId" EXPECTED_PARAMETER_KEY = "expected_parameter_key" EXPECTED_PARAMETER_VALUE = "expected_parameter_value" +IS_TRUNCATE_OP = "__is_truncate_op__" diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb index e7c30a2c4dc1..b9f75369d76e 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_constants.rb @@ -95,3 +95,5 @@ EXPECTED_PARAMETER_VALUE = %q"expected_parameter_value" +IS_TRUNCATE_OP = %q"__is_truncate_op__" + diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 6f146f2d30eb..c73d17158da5 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -3362,3 +3362,4 @@ const string WRITE_ID = "writeId", // Keys for alter table environment context parameters const string EXPECTED_PARAMETER_KEY = "expected_parameter_key", const string EXPECTED_PARAMETER_VALUE = "expected_parameter_value", +const string IS_TRUNCATE_OP = "__is_truncate_op__" diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index ed87a33427a8..2595da5e42e3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@ -36,7 +36,7 @@ public interface AlterHandler extends Configurable { /** * @deprecated As of release 2.2.0. Replaced by {@link #alterTable(RawStore, Warehouse, String, - * String, String, Table, EnvironmentContext, IHMSHandler, String, boolean)} + * String, String, Table, EnvironmentContext, IHMSHandler, String)} * * handles alter table, the changes could be cascaded to partitions if applicable * @@ -64,7 +64,7 @@ public interface AlterHandler extends Configurable { default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, String name, Table newTable, EnvironmentContext envContext) throws InvalidOperationException, MetaException { - alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null, null, false); + alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null, null); } /** @@ -85,7 +85,6 @@ default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbna * @param handler * HMSHandle object (required to log event notification) * @param writeIdList write id list for the table - * @param isTruncateOp boolean flag to specify if this is truncate operation * @param envContext environment context variable * @throws InvalidOperationException * thrown if the newTable object is invalid @@ -94,12 +93,12 @@ default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbna */ void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, String name, Table newTable, EnvironmentContext envContext, - IHMSHandler handler, String writeIdList, boolean isTruncateOp) + IHMSHandler handler, String writeIdList) throws InvalidOperationException, MetaException; /** * @deprecated As of release 2.2.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, - * String, String, List, EnvironmentContext, String, long, IHMSHandler, boolean)} + * String, String, List, EnvironmentContext, String, long, IHMSHandler)} * * handles alter partition * @@ -162,7 +161,7 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName /** * @deprecated As of release 3.0.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, - * String, String, List, EnvironmentContext, String, long, IHMSHandler, boolean)} + * String, String, List, EnvironmentContext, String, long, IHMSHandler)} * * handles alter partitions * @@ -200,7 +199,6 @@ List alterPartitions(final RawStore msdb, Warehouse wh, * @param writeIdList write id list for the table * @param writeId writeId for the table * @param handler HMSHandle object (required to log event notification) - * @param isTruncateOp whether the operation is truncate * @return the altered partition list * @throws InvalidOperationException thrown if the operation is invalid * @throws InvalidObjectException thrown if the new_parts object is invalid @@ -210,6 +208,6 @@ List alterPartitions(final RawStore msdb, Warehouse wh, List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, String writeIdList, long writeId, - IHMSHandler handler, boolean isTruncateOp) + IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index e25dc2b48257..98f0676f27bb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -3375,7 +3375,7 @@ private void alterPartitionsForTruncate(RawStore ms, String catName, String dbNa } } alterHandler.alterPartitions(ms, wh, catName, dbName, tableName, partitions, environmentContext, - validWriteIds, writeId, this, true); + validWriteIds, writeId, this); } private void alterTableStatsForTruncate(RawStore ms, String catName, String dbName, @@ -3393,7 +3393,7 @@ private void alterTableStatsForTruncate(RawStore ms, String catName, String dbNa table.setWriteId(writeId); } alterHandler.alterTable(ms, wh, catName, dbName, tableName, table, - environmentContext, this, validWriteIds, true); + environmentContext, this, validWriteIds); } return; } @@ -6108,7 +6108,7 @@ private void alter_partitions_with_environment_context(String catName, String db firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table, null, tmpPart, this)); } oldParts = alterHandler.alterPartitions(getMS(), wh, catName, db_name, tbl_name, new_parts, - environmentContext, writeIdList, writeId, this, false); + environmentContext, writeIdList, writeId, this); Iterator olditr = oldParts.iterator(); for (Partition tmpPart : new_parts) { @@ -6260,7 +6260,7 @@ private void alter_table_core(String catName, String dbname, String name, Table } firePreEvent(new PreAlterTableEvent(oldt, newTable, this)); alterHandler.alterTable(getMS(), wh, catName, dbname, name, newTable, - envContext, this, validWriteIdList, false); + envContext, this, validWriteIdList); success = true; } catch (Exception e) { ex = e; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index c824cd6f5edf..d3765e5b2cbb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -101,7 +101,7 @@ public void setConf(Configuration conf) { @Override public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, String name, Table newt, EnvironmentContext environmentContext, - IHMSHandler handler, String writeIdList, boolean isTruncateOp) + IHMSHandler handler, String writeIdList) throws InvalidOperationException, MetaException { catName = normalizeIdentifier(catName); name = normalizeIdentifier(name); @@ -110,12 +110,18 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam final boolean cascade; final boolean replDataLocationChanged; final boolean isReplicated; + final boolean isTruncateOp; if ((environmentContext != null) && environmentContext.isSetProperties()) { cascade = StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(StatsSetupConst.CASCADE)); replDataLocationChanged = ReplConst.TRUE.equals(environmentContext.getProperties().get(ReplConst.REPL_DATA_LOCATION_CHANGED)); + isTruncateOp = environmentContext.getProperties() + .containsKey(hive_metastoreConstants.IS_TRUNCATE_OP) ? Boolean.TRUE.toString() + .equalsIgnoreCase(environmentContext.getProperties() + .get(hive_metastoreConstants.IS_TRUNCATE_OP)) : false; } else { cascade = false; replDataLocationChanged = false; + isTruncateOp = false; } if (newt == null) { @@ -800,7 +806,7 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final EnvironmentContext environmentContext) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { return alterPartitions(msdb, wh, MetaStoreUtils.getDefaultCatalog(conf), dbname, name, new_parts, - environmentContext, null, -1, null, false); + environmentContext, null, -1, null); } private Map, Partition> getExistingPartitions(final RawStore msdb, @@ -828,7 +834,7 @@ private Map, Partition> getExistingPartitions(final RawStore msdb, public List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, String writeIdList, long writeId, - IHMSHandler handler, boolean isTruncateOp) + IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { List oldParts = new ArrayList<>(); List> partValsList = new ArrayList<>(); @@ -836,6 +842,15 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); } + final boolean isTruncateOp; + if ((environmentContext != null) && environmentContext.isSetProperties()) { + isTruncateOp = environmentContext.getProperties() + .containsKey(hive_metastoreConstants.IS_TRUNCATE_OP) ? Boolean.TRUE.toString() + .equalsIgnoreCase(environmentContext.getProperties() + .get(hive_metastoreConstants.IS_TRUNCATE_OP)) : false; + } else { + isTruncateOp = false; + } boolean success = false; try { From 64479b10aed8dc278fd13816811f18e424bfc948 Mon Sep 17 00:00:00 2001 From: saihemanth Date: Wed, 22 Jan 2025 11:40:14 +0530 Subject: [PATCH 4/6] Address test failures --- .../main/java/org/apache/hadoop/hive/metastore/HMSHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index 98f0676f27bb..c41edccc8149 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -3357,6 +3357,7 @@ private void updateStatsForTruncate(Map props, EnvironmentContext StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE); environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); environmentContext.putToProperties(StatsSetupConst.DO_NOT_POPULATE_QUICK_STATS, StatsSetupConst.TRUE); + environmentContext.putToProperties(hive_metastoreConstants.IS_TRUNCATE_OP, Boolean.TRUE.toString()); //then invalidate column stats StatsSetupConst.clearColumnStatsState(props); return; From c6d24026d7b30b675bf78fdfb0347a28a5c31119 Mon Sep 17 00:00:00 2001 From: saihemanth Date: Thu, 23 Jan 2025 17:31:58 +0530 Subject: [PATCH 5/6] Address test failures --- .../ql/parse/repl/load/message/TruncatePartitionHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java index 5db0ef990533..341752de3665 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java @@ -47,7 +47,8 @@ public List> handle(Context context) throws SemanticException { AlterPartitionsMessage singleMsg = deserializer.getAlterPartitionsMessage( context.dmd.getPayload()); tblObj = singleMsg.getTableObj(); - tName = TableName.fromString(tblObj.getTableName(), null, tblObj.getDbName()); + tName = TableName.fromString(singleMsg.getTable(), null, + context.isDbNameEmpty() ? singleMsg.getDB() : context.dbName); List> afterPartitionsList = singleMsg.getPartitions(); List> childTaskList = new ArrayList<>(); for(Map afterIteratorMap : afterPartitionsList) { From a1fc677cc6c54faaccab51a38407ed4625c5a48c Mon Sep 17 00:00:00 2001 From: saihemanth Date: Fri, 24 Jan 2025 19:04:52 +0530 Subject: [PATCH 6/6] Address review comments --- .../hadoop/hive/metastore/HMSHandler.java | 37 +++++++++++++++++-- .../hive/metastore/HiveAlterHandler.java | 13 +------ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index c41edccc8149..bbc3b0a1c386 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -3357,7 +3357,6 @@ private void updateStatsForTruncate(Map props, EnvironmentContext StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE); environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); environmentContext.putToProperties(StatsSetupConst.DO_NOT_POPULATE_QUICK_STATS, StatsSetupConst.TRUE); - environmentContext.putToProperties(hive_metastoreConstants.IS_TRUNCATE_OP, Boolean.TRUE.toString()); //then invalidate column stats StatsSetupConst.clearColumnStatsState(props); return; @@ -3369,14 +3368,45 @@ private void alterPartitionsForTruncate(RawStore ms, String catName, String dbNa if (partitions.isEmpty()) { return; } + List> partValsList = new ArrayList<>(); for (Partition partition: partitions) { updateStatsForTruncate(partition.getParameters(), environmentContext); if (writeId > 0) { partition.setWriteId(writeId); } + partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System + .currentTimeMillis() / 1000)); + partValsList.add(partition.getValues()); + } + ms.alterPartitions(catName, dbName, tableName, partValsList, partitions, writeId, validWriteIds); + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + boolean shouldSendSingleEvent = MetastoreConf.getBoolVar(this.getConf(), + MetastoreConf.ConfVars.NOTIFICATION_ALTER_PARTITIONS_V2_ENABLED); + if (shouldSendSingleEvent) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITIONS, + new AlterPartitionsEvent(partitions, partitions, table, true, true, this), environmentContext); + } else { + for (Partition partition : partitions) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITION, + new AlterPartitionEvent(partition, partition, table, true, true, partition.getWriteId(), this), + environmentContext); + } + } + } + if (listeners != null && !listeners.isEmpty()) { + boolean shouldSendSingleEvent = MetastoreConf.getBoolVar(this.getConf(), + MetastoreConf.ConfVars.NOTIFICATION_ALTER_PARTITIONS_V2_ENABLED); + if (shouldSendSingleEvent) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ALTER_PARTITIONS, + new AlterPartitionsEvent(partitions, partitions, table, true, true, this), environmentContext); + } else { + for (Partition partition : partitions) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ALTER_PARTITION, + new AlterPartitionEvent(partition, partition, table, true, true, partition.getWriteId(), this), + environmentContext); + } + } } - alterHandler.alterPartitions(ms, wh, catName, dbName, tableName, partitions, environmentContext, - validWriteIds, writeId, this); } private void alterTableStatsForTruncate(RawStore ms, String catName, String dbName, @@ -3388,6 +3418,7 @@ private void alterTableStatsForTruncate(RawStore ms, String catName, String dbNa } else { EnvironmentContext environmentContext = new EnvironmentContext(); updateStatsForTruncate(table.getParameters(), environmentContext); + environmentContext.putToProperties(hive_metastoreConstants.IS_TRUNCATE_OP, Boolean.TRUE.toString()); // TODO: this should actually pass thru and set writeId for txn stats. if (writeId > 0) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index d3765e5b2cbb..47001a60ae0a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -842,15 +842,6 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); } - final boolean isTruncateOp; - if ((environmentContext != null) && environmentContext.isSetProperties()) { - isTruncateOp = environmentContext.getProperties() - .containsKey(hive_metastoreConstants.IS_TRUNCATE_OP) ? Boolean.TRUE.toString() - .equalsIgnoreCase(environmentContext.getProperties() - .get(hive_metastoreConstants.IS_TRUNCATE_OP)) : false; - } else { - isTruncateOp = false; - } boolean success = false; try { @@ -904,12 +895,12 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final MetastoreConf.ConfVars.NOTIFICATION_ALTER_PARTITIONS_V2_ENABLED); if (shouldSendSingleEvent) { MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITIONS, - new AlterPartitionsEvent(oldParts, new_parts, tbl, isTruncateOp, true, handler), environmentContext); + new AlterPartitionsEvent(oldParts, new_parts, tbl, false, true, handler), environmentContext); } else { for (Partition newPart : new_parts) { Partition oldPart = oldPartMap.get(newPart.getValues()); MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITION, - new AlterPartitionEvent(oldPart, newPart, tbl, isTruncateOp, true, newPart.getWriteId(), handler), + new AlterPartitionEvent(oldPart, newPart, tbl, false, true, newPart.getWriteId(), handler), environmentContext); } }