Merge branch 'master' of github.com:logicalclocks/feature-store-api i…

…nto the-merge-python-hsfs

hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/SparkEngine.java

@@ -1097,5 +1097,10 @@ protected String makeQueryName(String queryName, FeatureGroupBase featureGroup)
     }
     return queryName;
   }
-
+
+  public void closeSparkSession() {
+    if (getSparkSession() != null) {
+      getSparkSession().stop();
+    }
+  }
 }

hsfs/python/tests/constructor/test_join.py

@@ -55,7 +55,7 @@ def test_from_response_json_basic_info(self, mocker, backend_fixtures):
         assert len(j._on) == 0
         assert len(j._left_on) == 0
         assert len(j._right_on) == 0
-        assert j._join_type == "INNER"
+        assert j._join_type == "LEFT"
         assert j._prefix is None
 
     def test_from_response_json_left_join(self, mocker, backend_fixtures):

hsfs/python/tests/core/test_arrow_flight_client.py

@@ -77,9 +77,6 @@ def _arrange_featureview_mocks(self, mocker, backend_fixtures):
             "hsfs.core.feature_view_engine.FeatureViewEngine.get_batch_query",
             return_value=fg.select_all(),
         )
-        mocker.patch(
-            "hsfs.core.transformation_function_engine.TransformationFunctionEngine.populate_builtin_transformation_functions"
-        )
         mocker.patch("hsfs.engine.python.Engine._apply_transformation_function")
 
         # required for batch query

hsfs/python/tests/core/test_feature_view_engine.py

@@ -23,14 +23,15 @@
     feature_view,
     split_statistics,
     training_dataset,
-    transformation_function_attached,
 )
 from hsfs.client.exceptions import FeatureStoreException
 from hsfs.constructor import fs_query
 from hsfs.constructor.query import Query
 from hsfs.core import arrow_flight_client, feature_view_engine
 from hsfs.core.feature_descriptive_statistics import FeatureDescriptiveStatistics
+from hsfs.hopsworks_udf import udf
 from hsfs.storage_connector import BigQueryConnector, StorageConnector
+from hsfs.transformation_function import TransformationFunction
 
 
 engine.init("python")
@@ -95,9 +96,6 @@ def test_save(self, mocker):
             "hsfs.core.feature_view_engine.FeatureViewEngine._get_feature_view_url",
             return_value=feature_view_url,
         )
-        mock_attach_transformation = mocker.patch(
-            "hsfs.core.feature_view_engine.FeatureViewEngine.attach_transformation_function",
-        )
         mock_print = mocker.patch("builtins.print")
 
         fv_engine = feature_view_engine.FeatureViewEngine(
@@ -113,7 +111,6 @@ def test_save(self, mocker):
 
         # Assert
         assert mock_fv_api.return_value.post.call_count == 1
-        assert mock_attach_transformation.call_count == 1
         assert mock_print.call_count == 1
         assert mock_print.call_args[0][
             0
@@ -353,10 +350,7 @@ def test_get_name(self, mocker):
 
         mock_fv_api = mocker.patch("hsfs.core.feature_view_api.FeatureViewApi")
         mocker.patch(
-            "hsfs.core.transformation_function_engine.TransformationFunctionEngine.get_fv_attached_transformation_fn"
-        )
-        mock_attach_transformation = mocker.patch(
-            "hsfs.core.feature_view_engine.FeatureViewEngine.attach_transformation_function",
+            "hsfs.core.feature_view_engine.FeatureViewEngine.get_attached_transformation_fn"
         )
 
         fv_engine = feature_view_engine.FeatureViewEngine(
@@ -385,7 +379,6 @@ def test_get_name(self, mocker):
 
         # Assert
         assert mock_fv_api.return_value.get_by_name_version.call_count == 0
-        assert mock_attach_transformation.call_count == 2
         assert mock_fv_api.return_value.get_by_name.call_count == 1
         assert len(result) == 2
 
@@ -395,10 +388,7 @@ def test_get_name_version(self, mocker):
 
         mock_fv_api = mocker.patch("hsfs.core.feature_view_api.FeatureViewApi")
         mocker.patch(
-            "hsfs.core.transformation_function_engine.TransformationFunctionEngine.get_fv_attached_transformation_fn"
-        )
-        mock_attach_transformation = mocker.patch(
-            "hsfs.core.feature_view_engine.FeatureViewEngine.attach_transformation_function",
+            "hsfs.core.feature_view_engine.FeatureViewEngine.get_attached_transformation_fn"
         )
 
         fv_engine = feature_view_engine.FeatureViewEngine(
@@ -420,7 +410,6 @@ def test_get_name_version(self, mocker):
 
         # Assert
         assert mock_fv_api.return_value.get_by_name_version.call_count == 1
-        assert mock_attach_transformation.call_count == 1
         assert mock_fv_api.return_value.get_by_name.call_count == 0
 
     def test_delete_name(self, mocker):
@@ -566,40 +555,73 @@ def test_get_batch_query_string_pit_query(self, mocker):
         assert mock_fv_api.return_value.get_batch_query.call_count == 1
         assert mock_qc_api.return_value.construct_query.call_count == 1
 
-    def test_attach_transformation_function(self, mocker):
-        def testFunction():
-            print("Test")
+    def test_get_attached_transformation_fn(self, mocker):
+        # Arrange
+        feature_store_id = 99
 
-        tf = transformation_function_attached.TransformationFunctionAttached(
-            name="tf_name", transformation_function=testFunction
+        mock_fv_api = mocker.patch("hsfs.core.feature_view_api.FeatureViewApi")
+
+        fv_engine = feature_view_engine.FeatureViewEngine(
+            feature_store_id=feature_store_id
         )
-        mocker.patch("hsfs.core.feature_view_api.FeatureViewApi")
-        mocker.patch(
-            "hsfs.core.transformation_function_engine.TransformationFunctionEngine.get_fv_attached_transformation_fn",
-            return_value={"label": tf},
+
+        @udf(int)
+        def test2(col1):
+            return col1 + 1
+
+        tf = TransformationFunction(
+            featurestore_id=10,
+            hopsworks_udf=test2,
         )
+
+        mock_fv_api.return_value.get_attached_transformation_fn.return_value = [tf]
+
+        # Act
+        result = fv_engine.get_attached_transformation_fn(name="fv_name", version=1)
+
+        # Assert
+        assert result == [tf]
+        assert mock_fv_api.return_value.get_attached_transformation_fn.call_count == 1
+
+    def test_get_attached_transformation_fn_multiple(self, mocker):
+        # Arrange
         feature_store_id = 99
+
+        mock_fv_api = mocker.patch("hsfs.core.feature_view_api.FeatureViewApi")
+
         fv_engine = feature_view_engine.FeatureViewEngine(
             feature_store_id=feature_store_id
         )
-        fv = feature_view.FeatureView(
-            name="fv_name",
-            version=1,
-            query=query,
-            featurestore_id=feature_store_id,
+
+        @udf(int)
+        def test1(col1):
+            return col1 + 1
+
+        tf1 = TransformationFunction(
+            featurestore_id=10,
+            hopsworks_udf=test1,
         )
-        fv.schema = query.features
+
+        @udf(int)
+        def test2(col1):
+            return col1 + 2
+
+        tf2 = TransformationFunction(
+            featurestore_id=10,
+            hopsworks_udf=test2,
+        )
+
+        mock_fv_api.return_value.get_attached_transformation_fn.return_value = [
+            tf1,
+            tf2,
+        ]
 
         # Act
-        fv_engine.attach_transformation_function(fv)
+        result = fv_engine.get_attached_transformation_fn(name="fv_name", version=1)
 
         # Assert
-        id_feature = fv.schema[0]
-        label_feature = fv.schema[1]
-        assert id_feature.name == "id"
-        assert id_feature.transformation_function is None
-        assert label_feature.name == "label"
-        assert label_feature.transformation_function == tf
+        assert result == [tf1, tf2]
+        assert mock_fv_api.return_value.get_attached_transformation_fn.call_count == 1
 
     def test_create_training_dataset(self, mocker):
         # Arrange
@@ -1619,7 +1641,6 @@ def test_get_training_dataset_metadata(self, mocker):
         # Assert
         assert mock_fv_api.return_value.get_training_dataset_by_version.call_count == 1
         assert result.schema == fv.schema
-        assert result.transformation_functions == fv.transformation_functions
 
     def test_create_training_data_metadata(self, mocker):
         # Arrange