[FSTORE-1063] Hopsworks data preview should use arrow flight to retr…

…ieve data if available (#1461)

* [FSTORE-1063] Hopsworks data preview should use arrow flight to retrieve data if available (#1630)

* fix cherry pick

hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

@@ -21,9 +21,15 @@
   describe "cached feature groups" do
     context 'with valid project, featurestore service enabled' do
       before :all do
+        @enable_flyingduck = getVar("enable_flyingduck")
+
         with_valid_project
       end
 
+      after :all do
+        setVar("enable_flyingduck", @enable_flyingduck[:value])
+      end
+
       it "should be able to add a offline cached featuregroup to the featurestore" do
         project = get_project
         featurestore_id = get_featurestore_id(project.id)
@@ -247,8 +253,25 @@
         expect(parsed_json["features"].first["description"]).to eql("this description contains ' and ;'")
       end
 
-      it "should be able to preview a offline cached featuregroup in the featurestore" do
+      it "should be able to preview a offline cached featuregroup in the featurestore using hive" do
+        project = get_project
+        setVar("enable_flyingduck", "false")
+        create_session(project[:username], "Pass123")
+        featurestore_id = get_featurestore_id(project.id)
+        json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id)
+        parsed_json = JSON.parse(json_result)
+        expect_status_details(201)
+        featuregroup_id = parsed_json["id"]
+        preview_featuregroup_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s + "/preview"
+        get preview_featuregroup_endpoint
+        parsed_json = JSON.parse(response.body)
+        expect_status_details(200)
+      end
+
+      it "should be able to preview a offline cached featuregroup in the featurestore using flying duck" do
         project = get_project
+        setVar("enable_flyingduck", "true")
+        create_session(project[:username], "Pass123")
         featurestore_id = get_featurestore_id(project.id)
         json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id)
         parsed_json = JSON.parse(json_result)

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupPreviewResource.java

@@ -109,7 +109,6 @@ public Response getPreview(@BeanParam FeatureGroupPreviewBeanParam featureGroupP
     }
 
     PreviewDTO previewDTO = previewBuilder.build(uriInfo, user, project, featuregroup,
-        featureGroupPreviewBeanParam.getPartition(),
         online,
         featureGroupPreviewBeanParam.getLimit() == null ? 20 : featureGroupPreviewBeanParam.getLimit());
 

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/PreviewBuilder.java

@@ -57,12 +57,12 @@ private URI uri(UriInfo uriInfo, Project project, Featuregroup featuregroup) {
   }
 
   public PreviewDTO build(UriInfo uriInfo, Users user, Project project, Featuregroup featuregroup,
-                          String partition, boolean online, int limit)
+                          boolean online, int limit)
       throws FeaturestoreException, HopsSecurityException {
 
     FeaturegroupPreview preview;
     try {
-      preview = featuregroupController.getFeaturegroupPreview(featuregroup, project, user, partition, online, limit);
+      preview = featuregroupController.getFeaturegroupPreview(featuregroup, project, user, online, limit);
     } catch (SQLException e) {
       throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_PREVIEW_FEATUREGROUP,
           Level.SEVERE, "Feature Group id: " + featuregroup.getId(), e.getMessage(), e);

hopsworks-common/pom.xml

@@ -248,6 +248,15 @@
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>flight-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>${projectlombok.version}</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -260,4 +269,4 @@
       </plugin>
     </plugins>
   </build>
-</project>
+</project>

hopsworks-common/src/main/java/io/hops/hopsworks/common/arrowflight/ArrowFlightConnectorDTO.java

@@ -0,0 +1,51 @@
+/*
+ * This file is part of Hopsworks
+ * Copyright (C) 2024, Hopsworks AB. All rights reserved
+ *
+ * Hopsworks is free software: you can redistribute it and/or modify it under the terms of
+ * the GNU Affero General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE.  See the GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License along with this program.
+ * If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package io.hops.hopsworks.common.arrowflight;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.NoArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+import java.util.Map;
+
+@AllArgsConstructor
+@NoArgsConstructor
+public class ArrowFlightConnectorDTO {
+
+  @Getter
+  @Setter
+  @JsonProperty(value = "type")
+  private String type;
+  @Getter
+  @Setter
+  @JsonProperty(value = "options")
+  private Map<String, String> options;
+  @Getter
+  @Setter
+  @JsonProperty(value = "query")
+  private String query;
+  @Getter
+  @Setter
+  @JsonProperty(value = "alias")
+  private String alias;
+  @Getter
+  @Setter
+  @JsonProperty(value = "filters")
+  private String filters;
+
+}