From e0a97cec2133e8560c7ba3bf7746f5da76e83867 Mon Sep 17 00:00:00 2001 From: Ralf Date: Wed, 29 May 2024 10:39:38 +0300 Subject: [PATCH] [FSTORE-1406] Delete Hive table only if it exists when deleting feature group (#1533) --- .../src/test/ruby/spec/featuregroup_spec.rb | 17 +++++++++ .../search/SearchFSCommandLogger.java | 38 +++++++++++-------- .../cached/OfflineFeatureGroupController.java | 15 +++++++- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb index f579576e47..baa0465a59 100644 --- a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb @@ -384,6 +384,23 @@ expect_status_details(200) end + it "should be able to delete a cached featuregroup from the featurestore if hive table was already deleted" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + # delete feature group offlinefs directory + delete "#{ENV['HOPSWORKS_API']}/project/#{project.id.to_s}/dataset//apps/hive/warehouse/#{project.projectname.downcase}_featurestore.db/#{parsed_json["name"]}_#{parsed_json["version"]}" + expect_status_details(204) + + featuregroup_id = parsed_json["id"] + delete_featuregroup_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s + delete delete_featuregroup_endpoint + expect_status_details(200) + end + it "should be able to clear the contents of a cached featuregroup in the featurestore" do project = get_project featurestore_id = get_featurestore_id(project.id) diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java index e9218afab6..43c4e5f7a4 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java @@ -64,82 +64,82 @@ public long count() { public void create(Featuregroup featureGroup) throws FeaturestoreException { SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.CREATE); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void create(FeatureView featureView) throws FeaturestoreException { SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.CREATE); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void create(TrainingDataset trainingDataset) throws FeaturestoreException { SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.CREATE); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateMetadata(Featuregroup featureGroup) throws FeaturestoreException { SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_METADATA); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateMetadata(FeatureView featureView) throws FeaturestoreException { SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_METADATA); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateMetadata(TrainingDataset trainingDataset) throws FeaturestoreException { SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_METADATA); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateKeywords(Featuregroup featureGroup) throws FeaturestoreException { SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_KEYWORDS); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateKeywords(FeatureView featureView) throws FeaturestoreException { SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_KEYWORDS); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateKeywords(TrainingDataset trainingDataset) throws FeaturestoreException { SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_KEYWORDS); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateTags(Featuregroup featureGroup) throws FeaturestoreException { SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_TAGS); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateTags(FeatureView featureView) throws FeaturestoreException { SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_TAGS); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void updateTags(TrainingDataset trainingDataset) throws FeaturestoreException { SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_TAGS); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void delete(Featuregroup featuregroup) throws FeaturestoreException { SearchFSCommand command = getCommand(featuregroup, SearchFSCommandOp.DELETE_ARTIFACT); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void delete(FeatureView featureView) throws FeaturestoreException { SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.DELETE_ARTIFACT); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void delete(TrainingDataset trainingDataset) throws FeaturestoreException { SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.DELETE_ARTIFACT); - commandFacade.persistAndFlush(command); + persistCommand(command); } public void delete(Project project) { SearchFSCommand command = getCommand(project, SearchFSCommandOp.DELETE_PROJECT); - commandFacade.persistAndFlush(command); + persistCommand(command); } private SearchFSCommand getCommand(Featuregroup featureGroup, SearchFSCommandOp op) throws FeaturestoreException { @@ -207,4 +207,10 @@ private SearchFSCommand getCommand(Project p, SearchFSCommandOp op) { command.setOp(op); return command; } + + private void persistCommand(SearchFSCommand command) { + if (command != null) { + commandFacade.persistAndFlush(command); + } + } } \ No newline at end of file diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java index eac4426ba3..1427ec45ae 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.AddDefaultConstraintRequest; import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; @@ -182,6 +183,8 @@ public void alterHiveTableFeatures(Featurestore featurestore, String tableName, public List getSchema(Featurestore featurestore, String tableName, Project project, Users user) throws FeaturestoreException { + List featureSchema = new ArrayList<>(); + String dbName = featurestoreController.getOfflineFeaturestoreDbName(featurestore.getProject()); ThriftHiveMetastore.Client client = getMetaStoreClient(project, user); Table table; @@ -191,6 +194,15 @@ public List getSchema(Featurestore featurestore, String table = getTable(client, dbName, tableName); schema = getFields(client, dbName, tableName); defaultConstraints = getDefaultConstraints(client, "hive", dbName, tableName); + } catch (FeaturestoreException e) { + if (e.getCause() instanceof NoSuchObjectException) { + // If the hive table doesn't exist return the feature group without features + // (Otherwise the user would not be able to get/delete the broken feature group in the UI/HSFS, + // since he would receive 500 responses to GET requests) + LOGGER.log(Level.SEVERE, "Feature group Hive table does not exist", e); + return featureSchema; + } + throw e; } finally { hiveController.finalizeMetastoreOperation(project, user, client); } @@ -198,7 +210,6 @@ public List getSchema(Featurestore featurestore, String // Setup a map of constraint values for easy access Map defaultConstraintsMap = defaultConstraints.stream() .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value)); - List featureSchema = new ArrayList<>(); // Partitions are listed first for (FieldSchema fieldSchema : table.getPartitionKeys()) { @@ -295,6 +306,8 @@ public void dropFeatureGroup(String dbName, String tableName, Project project, U try { client = hiveController.openUserMetastoreClient(project, user); client.drop_table(dbName, tableName, true); + } catch (NoSuchObjectException e) { + LOGGER.log(Level.INFO, "Hive table being deleted does not exist", e); } catch (TException e) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_DELETE_FEATUREGROUP, Level.SEVERE, "Error dropping feature group in the Hive Metastore: " + e.getMessage(), e.getMessage(), e);