[FSTORE-1406] Delete Hive table only if it exists when deleting featu…

…re group (#1533)

hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

@@ -384,6 +384,23 @@
         expect_status_details(200)
       end
 
+      it "should be able to delete a cached featuregroup from the featurestore if hive table was already deleted" do
+        project = get_project
+        featurestore_id = get_featurestore_id(project.id)
+        json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id)
+        parsed_json = JSON.parse(json_result)
+        expect_status_details(201)
+
+        # delete feature group offlinefs directory
+        delete "#{ENV['HOPSWORKS_API']}/project/#{project.id.to_s}/dataset//apps/hive/warehouse/#{project.projectname.downcase}_featurestore.db/#{parsed_json["name"]}_#{parsed_json["version"]}"
+        expect_status_details(204)
+
+        featuregroup_id = parsed_json["id"]
+        delete_featuregroup_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s
+        delete delete_featuregroup_endpoint
+        expect_status_details(200)
+      end
+
       it "should be able to clear the contents of a cached featuregroup in the featurestore" do
         project = get_project
         featurestore_id = get_featurestore_id(project.id)

hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java

@@ -64,82 +64,82 @@ public long count() {
 
   public void create(Featuregroup featureGroup) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.CREATE);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void create(FeatureView featureView) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.CREATE);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void create(TrainingDataset trainingDataset) throws FeaturestoreException {
     SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.CREATE);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateMetadata(Featuregroup featureGroup) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_METADATA);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateMetadata(FeatureView featureView) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_METADATA);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateMetadata(TrainingDataset trainingDataset) throws FeaturestoreException {
     SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_METADATA);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateKeywords(Featuregroup featureGroup) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_KEYWORDS);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateKeywords(FeatureView featureView) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_KEYWORDS);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateKeywords(TrainingDataset trainingDataset) throws FeaturestoreException {
     SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_KEYWORDS);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateTags(Featuregroup featureGroup) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_TAGS);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateTags(FeatureView featureView) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_TAGS);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void updateTags(TrainingDataset trainingDataset) throws FeaturestoreException {
     SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_TAGS);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void delete(Featuregroup featuregroup) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featuregroup, SearchFSCommandOp.DELETE_ARTIFACT);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void delete(FeatureView featureView) throws FeaturestoreException {
     SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.DELETE_ARTIFACT);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void delete(TrainingDataset trainingDataset) throws FeaturestoreException {
     SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.DELETE_ARTIFACT);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   public void delete(Project project) {
     SearchFSCommand command = getCommand(project, SearchFSCommandOp.DELETE_PROJECT);
-    commandFacade.persistAndFlush(command);
+    persistCommand(command);
   }
 
   private SearchFSCommand getCommand(Featuregroup featureGroup, SearchFSCommandOp op) throws FeaturestoreException {
@@ -207,4 +207,10 @@ private SearchFSCommand getCommand(Project p, SearchFSCommandOp op) {
     command.setOp(op);
     return command;
   }
+
+  private void persistCommand(SearchFSCommand command) {
+    if (command != null) {
+      commandFacade.persistAndFlush(command);
+    }
+  }
 }

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.hive.metastore.api.AddDefaultConstraintRequest;
 import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
 import org.apache.hadoop.hive.metastore.api.SkewedInfo;
@@ -182,6 +183,8 @@ public void alterHiveTableFeatures(Featurestore featurestore, String tableName,
 
   public List<FeatureGroupFeatureDTO> getSchema(Featurestore featurestore, String tableName,
                                                 Project project, Users user) throws FeaturestoreException {
+    List<FeatureGroupFeatureDTO> featureSchema = new ArrayList<>();
+
     String dbName = featurestoreController.getOfflineFeaturestoreDbName(featurestore.getProject());
     ThriftHiveMetastore.Client client = getMetaStoreClient(project, user);
     Table table;
@@ -191,14 +194,22 @@ public List<FeatureGroupFeatureDTO> getSchema(Featurestore featurestore, String
       table = getTable(client, dbName, tableName);
       schema = getFields(client, dbName, tableName);
       defaultConstraints = getDefaultConstraints(client, "hive", dbName, tableName);
+    } catch (FeaturestoreException e) {
+      if (e.getCause() instanceof NoSuchObjectException) {
+        // If the hive table doesn't exist return the feature group without features
+        // (Otherwise the user would not be able to get/delete the broken feature group in the UI/HSFS,
+        // since he would receive 500 responses to GET requests)
+        LOGGER.log(Level.SEVERE, "Feature group Hive table does not exist", e);
+        return featureSchema;
+      }
+      throw e;
     } finally {
       hiveController.finalizeMetastoreOperation(project, user, client);
     }
 
     // Setup a map of constraint values for easy access
     Map<String, String> defaultConstraintsMap = defaultConstraints.stream()
         .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value));
-    List<FeatureGroupFeatureDTO> featureSchema = new ArrayList<>();
 
     // Partitions are listed first
     for (FieldSchema fieldSchema : table.getPartitionKeys()) {
@@ -295,6 +306,8 @@ public void dropFeatureGroup(String dbName, String tableName, Project project, U
     try {
       client = hiveController.openUserMetastoreClient(project, user);
       client.drop_table(dbName, tableName, true);
+    } catch (NoSuchObjectException e) {
+      LOGGER.log(Level.INFO, "Hive table being deleted does not exist", e);
     } catch (TException e) {
       throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_DELETE_FEATUREGROUP, Level.SEVERE,
           "Error dropping feature group in the Hive Metastore: " +  e.getMessage(), e.getMessage(), e);