From 14048fd535f982eed7a3ba3af7d93557335e325a Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Thu, 6 Jan 2022 23:46:24 -0800 Subject: [PATCH 01/15] Compare only specs in integration tests (#2200) * Modify registry to_dict method to allow for returning only specs Signed-off-by: Felix Wang * Change test_cli test to keep only specs Signed-off-by: Felix Wang --- .../tests/integration/registration/test_cli.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/integration/registration/test_cli.py b/sdk/python/tests/integration/registration/test_cli.py index b92dc52642..f05674ea5c 100644 --- a/sdk/python/tests/integration/registration/test_cli.py +++ b/sdk/python/tests/integration/registration/test_cli.py @@ -44,6 +44,12 @@ def test_universal_cli(test_repo_config) -> None: fs = FeatureStore(repo_path=str(repo_path)) registry_dict = fs.registry.to_dict(project=project) + # Save only the specs, not the metadata. + registry_specs = { + key: [fco["spec"] for fco in value] + for key, value in registry_dict.items() + } + # entity & feature view list commands should succeed result = runner.run(["entities", "list"], cwd=repo_path) assertpy.assert_that(result.returncode).is_equal_to(0) @@ -83,8 +89,12 @@ def test_universal_cli(test_repo_config) -> None: ) # Confirm that registry contents have not changed. - assertpy.assert_that(registry_dict).is_equal_to( - fs.registry.to_dict(project=project) + registry_dict = fs.registry.to_dict(project=project) + assertpy.assert_that(registry_specs).is_equal_to( + { + key: [fco["spec"] for fco in value] + for key, value in registry_dict.items() + } ) result = runner.run(["teardown"], cwd=repo_path) From 4c32d759a930547597575e48c69dd5cb7f3151d5 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 7 Jan 2022 13:57:25 +0000 Subject: [PATCH 02/15] Refactor `OnlineResponse.to_dict()` (#2196) Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- sdk/python/feast/online_response.py | 33 +++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/online_response.py b/sdk/python/feast/online_response.py index 359e216165..8f30668f72 100644 --- a/sdk/python/feast/online_response.py +++ b/sdk/python/feast/online_response.py @@ -26,7 +26,6 @@ from feast.type_map import ( _proto_value_to_value_type, _python_value_to_proto_value, - feast_value_type_to_python_type, python_values_to_feast_value_type, ) from feast.value_type import ValueType @@ -63,14 +62,34 @@ def to_dict(self) -> Dict[str, Any]: """ Converts GetOnlineFeaturesResponse features into a dictionary form. """ + # Status for every Feature should be present in every record. features_dict: Dict[str, List[Any]] = { - k: list() for row in self.field_values for k, _ in row.statuses.items() + k: list() for k in self.field_values[0].statuses.keys() } - - for row in self.field_values: - for feature in features_dict.keys(): - native_type_value = feast_value_type_to_python_type(row.fields[feature]) - features_dict[feature].append(native_type_value) + rows = [record.fields for record in self.field_values] + + # Find the first non-null instance of each Feature to determine + # which ValueType. + val_types = {k: None for k in features_dict.keys()} + for feature in features_dict.keys(): + for row in rows: + try: + val_types[feature] = row[feature].WhichOneof("val") + except KeyError: + continue + if val_types[feature] is not None: + break + + # Now we know what attribute to fetch. + for feature, val_type in val_types.items(): + if val_type is None: + features_dict[feature] = [None] * len(rows) + else: + for row in rows: + val = getattr(row[feature], val_type) + if "_list_" in val_type: + val = list(val.val) + features_dict[feature].append(val) return features_dict From 30f7bbad96651a19c698663582eafa43779d313e Mon Sep 17 00:00:00 2001 From: corentinmarek Date: Fri, 7 Jan 2022 15:28:25 +0000 Subject: [PATCH 03/15] Add on demand feature views deletion (#2203) Signed-off-by: corentinmarek --- sdk/python/feast/registry.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index e57ecdee2c..d5c4c08048 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -667,6 +667,18 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): self.commit() return + for idx, existing_on_demand_feature_view_proto in enumerate( + self.cached_registry_proto.on_demand_feature_views + ): + if ( + existing_on_demand_feature_view_proto.spec.name == name + and existing_on_demand_feature_view_proto.spec.project == project + ): + del self.cached_registry_proto.on_demand_feature_views[idx] + if commit: + self.commit() + return + raise FeatureViewNotFoundException(name, project) def delete_entity(self, name: str, project: str, commit: bool = True): From f969e53d1eb34cdf993c852742da7ec9878f86aa Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 7 Jan 2022 16:28:25 +0000 Subject: [PATCH 04/15] Use FeatureViewProjection instead of FeatureView in ODFV (#2186) * Use FeatureViewProjection instead of FeatureView in ODFV Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Refactor `get_online_features` to allow for use of FeatureViewProjections in ODFVs. Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Refactor `get_online_features` to improve performance Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Fix linting error Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Refactor `_set_table_entity_keys` for clarity and performance Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Customer IDs should be `ValueType.STRING` Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Use Entity ValueType information in `_convert_arrow_to_proto` Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Only call `_augment_response_with_on_demand_transforms` if needed Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- protos/feast/core/OnDemandFeatureView.proto | 4 +- sdk/go/request.go | 10 +- sdk/go/request_test.go | 4 +- sdk/python/feast/cli.py | 5 +- sdk/python/feast/feature_store.py | 352 ++++++++---------- sdk/python/feast/feature_view.py | 2 +- .../feast/infra/passthrough_provider.py | 4 +- sdk/python/feast/infra/provider.py | 12 +- sdk/python/feast/on_demand_feature_view.py | 71 +++- sdk/python/feast/online_response.py | 77 +--- .../online_store/test_online_retrieval.py | 14 +- 11 files changed, 241 insertions(+), 314 deletions(-) diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 4cfac4cbd0..31fe90a9ba 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -24,6 +24,7 @@ option java_package = "feast.proto.core"; import "google/protobuf/timestamp.proto"; import "feast/core/FeatureView.proto"; +import "feast/core/FeatureViewProjection.proto"; import "feast/core/Feature.proto"; import "feast/core/DataSource.proto"; @@ -43,7 +44,7 @@ message OnDemandFeatureViewSpec { // List of features specifications for each feature defined with this feature view. repeated FeatureSpecV2 features = 3; - // List of features specifications for each feature defined with this feature view. + // Map of inputs for this feature view. map inputs = 4; UserDefinedFunction user_defined_function = 5; @@ -59,6 +60,7 @@ message OnDemandFeatureViewMeta { message OnDemandInput { oneof input { FeatureView feature_view = 1; + FeatureViewProjection feature_view_projection = 3; DataSource request_data_source = 2; } } diff --git a/sdk/go/request.go b/sdk/go/request.go index e6da10ff9b..94fecea01b 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -35,12 +35,12 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest if err != nil { return nil, err } - if len(r.Entities) == 0 { - return nil, fmt.Errorf("Entities must be provided") - } + if len(r.Entities) == 0 { + return nil, fmt.Errorf("Entities must be provided") + } - firstRow := r.Entities[0] - columnSize := len(firstRow) + firstRow := r.Entities[0] + columnSize := len(firstRow) // build request entity rows from native entities entityColumns := make(map[string][]*types.Value, columnSize) diff --git a/sdk/go/request_test.go b/sdk/go/request_test.go index 6beb15f7f7..9122c8ca40 100644 --- a/sdk/go/request_test.go +++ b/sdk/go/request_test.go @@ -37,12 +37,12 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { }, }, Entities: map[string]*types.RepeatedValue{ - "entity1": &types.RepeatedValue{ + "entity1": { Val: []*types.Value{ Int64Val(1), Int64Val(1), Int64Val(1), }, }, - "entity2": &types.RepeatedValue{ + "entity2": { Val: []*types.Value{ StrVal("bob"), StrVal("annie"), StrVal("jane"), }, diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 186d0185ef..4950977e2a 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -284,9 +284,8 @@ def feature_view_list(ctx: click.Context): if isinstance(feature_view, FeatureView): entities.update(feature_view.entities) elif isinstance(feature_view, OnDemandFeatureView): - for backing_fv in feature_view.inputs.values(): - if isinstance(backing_fv, FeatureView): - entities.update(backing_fv.entities) + for backing_fv in feature_view.input_feature_view_projections.values(): + entities.update(store.get_feature_view(backing_fv.name).entities) table.append( [ feature_view.name, diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index d43a114b36..f1fee70336 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -15,7 +15,7 @@ import itertools import os import warnings -from collections import Counter, OrderedDict, defaultdict +from collections import Counter, defaultdict from datetime import datetime from pathlib import Path from typing import ( @@ -62,14 +62,14 @@ ) from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.on_demand_feature_view import OnDemandFeatureView -from feast.online_response import OnlineResponse, _infer_online_entity_rows +from feast.online_response import OnlineResponse from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.serving.ServingService_pb2 import ( FieldStatus, - GetOnlineFeaturesRequestV2, GetOnlineFeaturesResponse, ) from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value from feast.registry import Registry from feast.repo_config import RepoConfig, load_repo_config from feast.request_feature_view import RequestFeatureView @@ -1073,6 +1073,15 @@ def get_online_features( set_usage_attribute("odfv", bool(grouped_odfv_refs)) set_usage_attribute("request_fv", bool(grouped_request_fv_refs)) + # All requested features should be present in the result. + requested_result_row_names = { + feat_ref.replace(":", "__") for feat_ref in _feature_refs + } + if not full_feature_names: + requested_result_row_names = { + name.rpartition("__")[-1] for name in requested_result_row_names + } + feature_views = list(view for view, _ in grouped_refs) entityless_case = DUMMY_ENTITY_NAME in [ entity_name @@ -1083,8 +1092,10 @@ def get_online_features( provider = self._get_provider() entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) entity_name_to_join_key_map: Dict[str, str] = {} + join_key_to_entity_type_map: Dict[str, ValueType] = {} for entity in entities: entity_name_to_join_key_map[entity.name] = entity.join_key + join_key_to_entity_type_map[entity.join_key] = entity.value_type for feature_view in requested_feature_views: for entity_name in feature_view.entities: entity = self._registry.get_entity( @@ -1099,13 +1110,14 @@ def get_online_features( entity.join_key, entity.join_key ) entity_name_to_join_key_map[entity_name] = join_key + join_key_to_entity_type_map[join_key] = entity.value_type needed_request_data, needed_request_fv_features = self.get_needed_request_data( grouped_odfv_refs, grouped_request_fv_refs ) join_key_rows = [] - request_data_features: Dict[str, List[Any]] = {} + request_data_features: Dict[str, List[Any]] = defaultdict(list) # Entity rows may be either entities or request data. for row in entity_rows: join_key_row = {} @@ -1115,17 +1127,21 @@ def get_online_features( entity_name in needed_request_data or entity_name in needed_request_fv_features ): - if entity_name not in request_data_features: - request_data_features[entity_name] = [] + if entity_name in needed_request_fv_features: + # If the data was requested as a feature then + # make sure it appears in the result. + requested_result_row_names.add(entity_name) request_data_features[entity_name].append(entity_value) - continue - try: - join_key = entity_name_to_join_key_map[entity_name] - except KeyError: - raise EntityNotFoundException(entity_name, self.project) - join_key_row[join_key] = entity_value - if entityless_case: - join_key_row[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL + else: + try: + join_key = entity_name_to_join_key_map[entity_name] + except KeyError: + raise EntityNotFoundException(entity_name, self.project) + # All join keys should be returned in the result. + requested_result_row_names.add(join_key) + join_key_row[join_key] = entity_value + if entityless_case: + join_key_row[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL if len(join_key_row) > 0: # May be empty if this entity row was request data join_key_rows.append(join_key_row) @@ -1134,88 +1150,111 @@ def get_online_features( needed_request_data, needed_request_fv_features, request_data_features ) - entity_row_proto_list = _infer_online_entity_rows(join_key_rows) - - union_of_entity_keys: List[EntityKeyProto] = [] - result_rows: List[GetOnlineFeaturesResponse.FieldValues] = [] + # Convert join_key_rows from rowise to columnar. + join_key_python_values: Dict[str, List[Value]] = defaultdict(list) + for join_key_row in join_key_rows: + for join_key, value in join_key_row.items(): + join_key_python_values[join_key].append(value) - for entity_row_proto in entity_row_proto_list: - # Create a list of entity keys to filter down for each feature view at lookup time. - union_of_entity_keys.append(_entity_row_to_key(entity_row_proto)) - # Also create entity values to append to the result - result_rows.append(_entity_row_to_field_values(entity_row_proto)) + # Convert all join key values to Protobuf Values + join_key_proto_values = { + k: python_values_to_proto_values(v, join_key_to_entity_type_map[k]) + for k, v in join_key_python_values.items() + } - # Keep track of what has been requested from the OnlineStore - # to avoid requesting the same thing twice for ODFVs. - retrieved_feature_refs: Set[str] = set() + # Populate result rows with join keys + result_rows = [ + GetOnlineFeaturesResponse.FieldValues() for _ in range(len(entity_rows)) + ] + for key, values in join_key_proto_values.items(): + for row_idx, result_row in enumerate(result_rows): + result_row.fields[key].CopyFrom(values[row_idx]) + result_row.statuses[key] = FieldStatus.PRESENT + + # Initialize the set of EntityKeyProtos once and reuse them for each FeatureView + # to avoid initialization overhead. + entity_keys = [EntityKeyProto() for _ in range(len(join_key_rows))] for table, requested_features in grouped_refs: - table_join_keys = [ - entity_name_to_join_key_map[entity_name] - for entity_name in table.entities - ] + # Get the correct set of entity values with the correct join keys. + entity_values = self._get_table_entity_values( + table, entity_name_to_join_key_map, join_key_proto_values, + ) + + # Set the EntityKeyProtos inplace. + self._set_table_entity_keys( + entity_values, entity_keys, + ) + + # Populate the result_rows with the Features from the OnlineStore inplace. self._populate_result_rows_from_feature_view( - table_join_keys, + entity_keys, full_feature_names, provider, requested_features, result_rows, table, - union_of_entity_keys, ) - table_feature_names = {feature.name for feature in table.features} - retrieved_feature_refs |= { - f"{table.name}:{feature}" if feature in table_feature_names else feature - for feature in requested_features - } - requested_result_row_names = self._get_requested_result_fields( - result_rows, needed_request_fv_features - ) - self._populate_odfv_dependencies( - entity_name_to_join_key_map, - full_feature_names, - grouped_odfv_refs, - provider, - request_data_features, - result_rows, - union_of_entity_keys, - retrieved_feature_refs, + self._populate_request_data_features( + request_data_features, result_rows, ) - self._augment_response_with_on_demand_transforms( - _feature_refs, - requested_result_row_names, - requested_on_demand_feature_views, - full_feature_names, - result_rows, + if grouped_odfv_refs: + self._augment_response_with_on_demand_transforms( + _feature_refs, + requested_on_demand_feature_views, + full_feature_names, + result_rows, + ) + + self._drop_unneeded_columns( + requested_result_row_names, result_rows, ) return OnlineResponse(GetOnlineFeaturesResponse(field_values=result_rows)) - def _get_requested_result_fields( - self, - result_rows: List[GetOnlineFeaturesResponse.FieldValues], - needed_request_fv_features: Set[str], - ): - # Get requested feature values so we can drop odfv dependencies that aren't requested - requested_result_row_names: Set[str] = set() - for result_row in result_rows: - for feature_name in result_row.fields.keys(): - requested_result_row_names.add(feature_name) - # Request feature view values are also request data features that should be in the - # final output - requested_result_row_names.update(needed_request_fv_features) - return requested_result_row_names - - def _populate_odfv_dependencies( - self, + @staticmethod + def _get_table_entity_values( + table: FeatureView, entity_name_to_join_key_map: Dict[str, str], - full_feature_names: bool, - grouped_odfv_refs: List[Tuple[OnDemandFeatureView, List[str]]], - provider: Provider, + join_key_proto_values: Dict[str, List[Value]], + ) -> Dict[str, List[Value]]: + # The correct join_keys expected by the OnlineStore for this Feature View. + table_join_keys = [ + entity_name_to_join_key_map[entity_name] for entity_name in table.entities + ] + + # If the FeatureView has a Projection then the join keys may be aliased. + alias_to_join_key_map = {v: k for k, v in table.projection.join_key_map.items()} + + # Subset to columns which are relevant to this FeatureView and + # give them the correct names. + entity_values = { + alias_to_join_key_map.get(k, k): v + for k, v in join_key_proto_values.items() + if alias_to_join_key_map.get(k, k) in table_join_keys + } + return entity_values + + @staticmethod + def _set_table_entity_keys( + entity_values: Dict[str, List[Value]], entity_keys: List[EntityKeyProto], + ): + """ + This method sets the a list of EntityKeyProtos inplace. + """ + keys = entity_values.keys() + # Columar to rowise (dict keys and values are guaranteed to have the same order). + rowise_values = zip(*entity_values.values()) + for entity_key in entity_keys: + # Make sure entity_keys are empty before setting. + entity_key.Clear() + entity_key.join_keys.extend(keys) + entity_key.entity_values.extend(next(rowise_values)) + + @staticmethod + def _populate_request_data_features( request_data_features: Dict[str, List[Any]], result_rows: List[GetOnlineFeaturesResponse.FieldValues], - union_of_entity_keys: List[EntityKeyProto], - retrieved_feature_refs: Set[str], ): # Add more feature values to the existing result rows for the request data features for feature_name, feature_values in request_data_features.items(): @@ -1228,39 +1267,8 @@ def _populate_odfv_dependencies( result_row.fields[feature_name].CopyFrom(proto_value) result_row.statuses[feature_name] = FieldStatus.PRESENT - # Add data if odfv requests specific feature views as dependencies - if len(grouped_odfv_refs) > 0: - for odfv, _ in grouped_odfv_refs: - for fv in odfv.input_feature_views.values(): - # Find the set of required Features which have not yet - # been retrieved. - not_yet_retrieved = { - feature.name - for feature in fv.projection.features - if f"{fv.name}:{feature.name}" not in retrieved_feature_refs - } - # If there are required Features which have not yet been retrieved - # retrieve them. - if not_yet_retrieved: - table_join_keys = [ - entity_name_to_join_key_map[entity_name] - for entity_name in fv.entities - ] - self._populate_result_rows_from_feature_view( - table_join_keys, - full_feature_names, - provider, - list(not_yet_retrieved), - result_rows, - fv, - union_of_entity_keys, - ) - # Update the set of retrieved Features with any newly retrieved - # Features. - retrieved_feature_refs |= not_yet_retrieved - + @staticmethod def get_needed_request_data( - self, grouped_odfv_refs: List[Tuple[OnDemandFeatureView, List[str]]], grouped_request_fv_refs: List[Tuple[RequestFeatureView, List[str]]], ) -> Tuple[Set[str], Set[str]]: @@ -1274,8 +1282,8 @@ def get_needed_request_data( needed_request_fv_features.add(feature.name) return needed_request_data, needed_request_fv_features + @staticmethod def ensure_request_data_values_exist( - self, needed_request_data: Set[str], needed_request_fv_features: Set[str], request_data_features: Dict[str, List[Any]], @@ -1296,17 +1304,13 @@ def ensure_request_data_values_exist( def _populate_result_rows_from_feature_view( self, - table_join_keys: List[str], + entity_keys: List[EntityKeyProto], full_feature_names: bool, provider: Provider, requested_features: List[str], result_rows: List[GetOnlineFeaturesResponse.FieldValues], table: FeatureView, - union_of_entity_keys: List[EntityKeyProto], ): - entity_keys = _get_table_entity_keys( - table, union_of_entity_keys, table_join_keys - ) read_rows = provider.online_read( config=self.config, table=table, @@ -1339,10 +1343,9 @@ def _populate_result_rows_from_feature_view( ) result_row.statuses[feature_ref] = FieldStatus.PRESENT + @staticmethod def _augment_response_with_on_demand_transforms( - self, feature_refs: List[str], - requested_result_row_names: Set[str], requested_on_demand_feature_views: List[OnDemandFeatureView], full_feature_names: bool, result_rows: List[GetOnlineFeaturesResponse.FieldValues], @@ -1350,22 +1353,17 @@ def _augment_response_with_on_demand_transforms( """Computes on demand feature values and adds them to the result rows. Assumes that 'result_rows' already contains the necessary request data and input feature - views for the on demand feature views. Unneeded feature values such as request data and - unrequested input feature views will be removed from 'result_rows'. + views for the on demand feature views. Args: feature_refs: List of all feature references to be returned. - requested_result_row_names: Fields from 'result_rows' that have been requested, and - therefore should not be dropped. + requested_on_demand_feature_views: List of all odfvs that have been requested. full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to "customer_fv__daily_transactions"). result_rows: List of result rows to be augmented with on demand feature values. """ - if len(requested_on_demand_feature_views) == 0: - return - requested_odfv_map = { odfv.name: odfv for odfv in requested_on_demand_feature_views } @@ -1414,11 +1412,25 @@ def _augment_response_with_on_demand_transforms( ) result_row.statuses[transformed_feature] = FieldStatus.PRESENT + @staticmethod + def _drop_unneeded_columns( + requested_result_row_names: Set[str], + result_rows: List[GetOnlineFeaturesResponse.FieldValues], + ): + """ + Unneeded feature values such as request data and unrequested input feature views will + be removed from 'result_rows'. + + Args: + requested_result_row_names: Fields from 'result_rows' that have been requested, and + therefore should not be dropped. + result_rows: List of result rows to be editted inplace. + """ # Drop values that aren't needed unneeded_features = [ val for val in result_rows[0].fields - if val not in requested_result_row_names and val not in odfv_result_names + if val not in requested_result_row_names ] for row_idx in range(len(result_rows)): result_row = result_rows[row_idx] @@ -1467,9 +1479,13 @@ def _get_feature_views_to_use( request_fvs[fv_name].with_projection(copy.copy(projection)) ) elif fv_name in od_fvs: - od_fvs_to_use.append( - od_fvs[fv_name].with_projection(copy.copy(projection)) - ) + odfv = od_fvs[fv_name].with_projection(copy.copy(projection)) + od_fvs_to_use.append(odfv) + # Let's make sure to include an FVs which the ODFV requires Features from. + for projection in odfv.input_feature_view_projections.values(): + fv = fvs[projection.name].with_projection(copy.copy(projection)) + if fv not in fvs_to_use: + fvs_to_use.append(fv) else: raise ValueError( f"The provided feature service {features.name} contains a reference to a feature view" @@ -1512,22 +1528,6 @@ def serve_transformations(self, port: int) -> None: transformation_server.start_server(self, port) -def _entity_row_to_key(row: GetOnlineFeaturesRequestV2.EntityRow) -> EntityKeyProto: - names, values = zip(*row.fields.items()) - return EntityKeyProto(join_keys=names, entity_values=values) - - -def _entity_row_to_field_values( - row: GetOnlineFeaturesRequestV2.EntityRow, -) -> GetOnlineFeaturesResponse.FieldValues: - result = GetOnlineFeaturesResponse.FieldValues() - for k in row.fields: - result.fields[k].CopyFrom(row.fields[k]) - result.statuses[k] = FieldStatus.PRESENT - - return result - - def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): collided_feature_refs = [] @@ -1581,21 +1581,27 @@ def _group_feature_refs( } # view name to feature names - views_features = defaultdict(list) - request_views_features = defaultdict(list) + views_features = defaultdict(set) + request_views_features = defaultdict(set) request_view_refs = set() # on demand view name to feature names - on_demand_view_features = defaultdict(list) + on_demand_view_features = defaultdict(set) for ref in features: view_name, feat_name = ref.split(":") if view_name in view_index: - views_features[view_name].append(feat_name) + views_features[view_name].add(feat_name) elif view_name in on_demand_view_index: - on_demand_view_features[view_name].append(feat_name) + on_demand_view_features[view_name].add(feat_name) + # Let's also add in any FV Feature dependencies here. + for input_fv_projection in on_demand_view_index[ + view_name + ].input_feature_view_projections.values(): + for input_feat in input_fv_projection.features: + views_features[input_fv_projection.name].add(input_feat.name) elif view_name in request_view_index: - request_views_features[view_name].append(feat_name) + request_views_features[view_name].add(feat_name) request_view_refs.add(ref) else: raise FeatureViewNotFoundException(view_name) @@ -1605,54 +1611,14 @@ def _group_feature_refs( request_fvs_result: List[Tuple[RequestFeatureView, List[str]]] = [] for view_name, feature_names in views_features.items(): - fvs_result.append((view_index[view_name], feature_names)) + fvs_result.append((view_index[view_name], list(feature_names))) for view_name, feature_names in request_views_features.items(): - request_fvs_result.append((request_view_index[view_name], feature_names)) + request_fvs_result.append((request_view_index[view_name], list(feature_names))) for view_name, feature_names in on_demand_view_features.items(): - odfvs_result.append((on_demand_view_index[view_name], feature_names)) + odfvs_result.append((on_demand_view_index[view_name], list(feature_names))) return fvs_result, odfvs_result, request_fvs_result, request_view_refs -def _get_table_entity_keys( - table: FeatureView, entity_keys: List[EntityKeyProto], table_join_keys: List[str] -) -> List[EntityKeyProto]: - reverse_join_key_map = { - alias: original for original, alias in table.projection.join_key_map.items() - } - required_entities = OrderedDict.fromkeys(sorted(table_join_keys)) - entity_key_protos = [] - for entity_key in entity_keys: - required_entities_to_values = required_entities.copy() - for i in range(len(entity_key.join_keys)): - entity_name = reverse_join_key_map.get( - entity_key.join_keys[i], entity_key.join_keys[i] - ) - entity_value = entity_key.entity_values[i] - - if entity_name in required_entities_to_values: - if required_entities_to_values[entity_name] is not None: - raise ValueError( - f"Duplicate entity keys detected. Table {table.name} expects {table_join_keys}. The entity " - f"{entity_name} was provided at least twice" - ) - required_entities_to_values[entity_name] = entity_value - - entity_names = [] - entity_values = [] - for entity_name, entity_value in required_entities_to_values.items(): - if entity_value is None: - raise ValueError( - f"Table {table.name} expects entity field {table_join_keys}. No entity value was found for " - f"{entity_name}" - ) - entity_names.append(entity_name) - entity_values.append(entity_value) - entity_key_protos.append( - EntityKeyProto(join_keys=entity_names, entity_values=entity_values) - ) - return entity_key_protos - - def _print_materialization_log( start_date, end_date, num_feature_views: int, online_store: str ): diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 1f31b192a3..57b60c0503 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -44,7 +44,7 @@ DUMMY_ENTITY_NAME = "__dummy" DUMMY_ENTITY_VAL = "" DUMMY_ENTITY = Entity( - name=DUMMY_ENTITY_NAME, join_key=DUMMY_ENTITY_ID, value_type=ValueType.INT32, + name=DUMMY_ENTITY_NAME, join_key=DUMMY_ENTITY_ID, value_type=ValueType.STRING, ) diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index b42f5b0daf..98937ce1fa 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -98,7 +98,7 @@ def ingest_df( if feature_view.batch_source.field_mapping is not None: table = _run_field_mapping(table, feature_view.batch_source.field_mapping) - join_keys = [entity.join_key for entity in entities] + join_keys = {entity.join_key: entity.value_type for entity in entities} rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys) self.online_write_batch( @@ -144,7 +144,7 @@ def materialize_single_feature_view( if feature_view.batch_source.field_mapping is not None: table = _run_field_mapping(table, feature_view.batch_source.field_mapping) - join_keys = [entity.join_key for entity in entities] + join_keys = {entity.join_key: entity.value_type for entity in entities} with tqdm_builder(table.num_rows) as pbar: for batch in table.to_batches(DEFAULT_BATCH_SIZE): diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 00591725fc..32bca8f7d7 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -300,21 +300,21 @@ def _coerce_datetime(ts): def _convert_arrow_to_proto( table: Union[pyarrow.Table, pyarrow.RecordBatch], feature_view: FeatureView, - join_keys: List[str], + join_keys: Dict[str, ValueType], ) -> List[Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]]: # Avoid ChunkedArrays which guarentees `zero_copy_only` availiable. if isinstance(table, pyarrow.Table): table = table.to_batches()[0] - columns = [(f.name, f.dtype) for f in feature_view.features] + [ - (key, ValueType.UNKNOWN) for key in join_keys - ] + columns = [(f.name, f.dtype) for f in feature_view.features] + list( + join_keys.items() + ) proto_values_by_column = { column: python_values_to_proto_values( - table.column(column).to_numpy(zero_copy_only=False), dtype + table.column(column).to_numpy(zero_copy_only=False), value_type ) - for column, dtype in columns + for column, value_type in columns } entity_keys = [ diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 0f55f0dde5..789422add4 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -6,10 +6,9 @@ import dill import pandas as pd -from feast import errors from feast.base_feature_view import BaseFeatureView from feast.data_source import RequestDataSource -from feast.errors import RegistryInferenceFailure +from feast.errors import RegistryInferenceFailure, SpecifiedFeaturesNotPresentError from feast.feature import Feature from feast.feature_view import FeatureView from feast.feature_view_projection import FeatureViewProjection @@ -45,8 +44,7 @@ class OnDemandFeatureView(BaseFeatureView): """ # TODO(adchia): remove inputs from proto and declaration - inputs: Dict[str, Union[FeatureView, RequestDataSource]] - input_feature_views: Dict[str, FeatureView] + input_feature_view_projections: Dict[str, FeatureViewProjection] input_request_data_sources: Dict[str, RequestDataSource] udf: MethodType @@ -55,21 +53,22 @@ def __init__( self, name: str, features: List[Feature], - inputs: Dict[str, Union[FeatureView, RequestDataSource]], + inputs: Dict[str, Union[FeatureView, FeatureViewProjection, RequestDataSource]], udf: MethodType, ): """ Creates an OnDemandFeatureView object. """ super().__init__(name, features) - self.inputs = inputs - self.input_feature_views = {} - self.input_request_data_sources = {} + self.input_feature_view_projections: Dict[str, FeatureViewProjection] = {} + self.input_request_data_sources: Dict[str, RequestDataSource] = {} for input_ref, odfv_input in inputs.items(): if isinstance(odfv_input, RequestDataSource): self.input_request_data_sources[input_ref] = odfv_input + elif isinstance(odfv_input, FeatureViewProjection): + self.input_feature_view_projections[input_ref] = odfv_input else: - self.input_feature_views[input_ref] = odfv_input + self.input_feature_view_projections[input_ref] = odfv_input.projection self.udf = udf @@ -79,11 +78,37 @@ def proto_class(self) -> Type[OnDemandFeatureViewProto]: def __copy__(self): fv = OnDemandFeatureView( - name=self.name, features=self.features, inputs=self.inputs, udf=self.udf + name=self.name, + features=self.features, + inputs=dict( + **self.input_feature_view_projections, **self.input_request_data_sources + ), + udf=self.udf, ) fv.projection = copy.copy(self.projection) return fv + def __eq__(self, other): + if not super().__eq__(other): + return False + + if ( + not self.input_feature_view_projections + == other.input_feature_view_projections + ): + return False + + if not self.input_request_data_sources == other.input_request_data_sources: + return False + + if not self.udf.__code__.co_code == other.udf.__code__.co_code: + return False + + return True + + def __hash__(self): + return super().__hash__() + def to_proto(self) -> OnDemandFeatureViewProto: """ Converts an on demand feature view object to its protobuf representation. @@ -95,8 +120,10 @@ def to_proto(self) -> OnDemandFeatureViewProto: if self.created_timestamp: meta.created_timestamp.FromDatetime(self.created_timestamp) inputs = {} - for input_ref, fv in self.input_feature_views.items(): - inputs[input_ref] = OnDemandInput(feature_view=fv.to_proto()) + for input_ref, fv_projection in self.input_feature_view_projections.items(): + inputs[input_ref] = OnDemandInput( + feature_view_projection=fv_projection.to_proto() + ) for input_ref, request_data_source in self.input_request_data_sources.items(): inputs[input_ref] = OnDemandInput( request_data_source=request_data_source.to_proto() @@ -132,6 +159,10 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): if on_demand_input.WhichOneof("input") == "feature_view": inputs[input_name] = FeatureView.from_proto( on_demand_input.feature_view + ).projection + elif on_demand_input.WhichOneof("input") == "feature_view_projection": + inputs[input_name] = FeatureViewProjection.from_proto( + on_demand_input.feature_view_projection ) else: inputs[input_name] = RequestDataSource.from_proto( @@ -177,9 +208,9 @@ def get_transformed_features_df( ) -> pd.DataFrame: # Apply on demand transformations columns_to_cleanup = [] - for input_fv in self.input_feature_views.values(): - for feature in input_fv.features: - full_feature_ref = f"{input_fv.name}__{feature.name}" + for input_fv_projection in self.input_feature_view_projections.values(): + for feature in input_fv_projection.features: + full_feature_ref = f"{input_fv_projection.name}__{feature.name}" if full_feature_ref in df_with_features.keys(): # Make sure the partial feature name is always present df_with_features[feature.name] = df_with_features[full_feature_ref] @@ -218,10 +249,12 @@ def infer_features(self): RegistryInferenceFailure: The set of features could not be inferred. """ df = pd.DataFrame() - for feature_view in self.input_feature_views.values(): - for feature in feature_view.features: + for feature_view_projection in self.input_feature_view_projections.values(): + for feature in feature_view_projection.features: dtype = feast_value_type_to_pandas_type(feature.dtype) - df[f"{feature_view.name}__{feature.name}"] = pd.Series(dtype=dtype) + df[f"{feature_view_projection.name}__{feature.name}"] = pd.Series( + dtype=dtype + ) df[f"{feature.name}"] = pd.Series(dtype=dtype) for request_data in self.input_request_data_sources.values(): for feature_name, feature_type in request_data.schema.items(): @@ -242,7 +275,7 @@ def infer_features(self): if specified_features not in inferred_features: missing_features.append(specified_features) if missing_features: - raise errors.SpecifiedFeaturesNotPresentError( + raise SpecifiedFeaturesNotPresentError( [f.name for f in missing_features], self.name ) else: diff --git a/sdk/python/feast/online_response.py b/sdk/python/feast/online_response.py index 8f30668f72..e6bf6be42c 100644 --- a/sdk/python/feast/online_response.py +++ b/sdk/python/feast/online_response.py @@ -12,23 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict -from typing import Any, Dict, List, cast +from typing import Any, Dict, List import pandas as pd from feast.feature_view import DUMMY_ENTITY_ID -from feast.protos.feast.serving.ServingService_pb2 import ( - GetOnlineFeaturesRequestV2, - GetOnlineFeaturesResponse, -) -from feast.protos.feast.types.Value_pb2 import Value as Value -from feast.type_map import ( - _proto_value_to_value_type, - _python_value_to_proto_value, - python_values_to_feast_value_type, -) -from feast.value_type import ValueType +from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesResponse class OnlineResponse: @@ -99,65 +88,3 @@ def to_df(self) -> pd.DataFrame: """ return pd.DataFrame(self.to_dict()) - - -def _infer_online_entity_rows( - entity_rows: List[Dict[str, Any]] -) -> List[GetOnlineFeaturesRequestV2.EntityRow]: - """ - Builds a list of EntityRow protos from Python native type format passed by user. - - Args: - entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair. - Returns: - A list of EntityRow protos parsed from args. - """ - - entity_rows_dicts = cast(List[Dict[str, Any]], entity_rows) - entity_row_list = [] - entity_type_map: Dict[str, ValueType] = dict() - entity_python_values_map = defaultdict(list) - - # Flatten keys-value dicts into lists for type inference - for entity in entity_rows_dicts: - for key, value in entity.items(): - if isinstance(value, Value): - inferred_type = _proto_value_to_value_type(value) - # If any ProtoValues were present their types must all be the same - if key in entity_type_map and entity_type_map.get(key) != inferred_type: - raise TypeError( - f"Input entity {key} has mixed types, {entity_type_map.get(key)} and {inferred_type}. That is not allowed." - ) - entity_type_map[key] = inferred_type - else: - entity_python_values_map[key].append(value) - - # Loop over all entities to infer dtype first in case of empty lists or nulls - for key, values in entity_python_values_map.items(): - inferred_type = python_values_to_feast_value_type(key, values) - - # If any ProtoValues were present their types must match the inferred type - if key in entity_type_map and entity_type_map.get(key) != inferred_type: - raise TypeError( - f"Input entity {key} has mixed types, {entity_type_map.get(key)} and {inferred_type}. That is not allowed." - ) - - entity_type_map[key] = inferred_type - - for entity in entity_rows_dicts: - fields = {} - for key, value in entity.items(): - if key not in entity_type_map: - raise ValueError( - f"field {key} cannot have all null values for type inference." - ) - - if isinstance(value, Value): - proto_value = value - else: - proto_value = _python_value_to_proto_value( - entity_type_map[key], [value] - )[0] - fields[key] = proto_value - entity_row_list.append(GetOnlineFeaturesRequestV2.EntityRow(fields=fields)) - return entity_row_list diff --git a/sdk/python/tests/integration/online_store/test_online_retrieval.py b/sdk/python/tests/integration/online_store/test_online_retrieval.py index b94f6f1772..265fedd282 100644 --- a/sdk/python/tests/integration/online_store/test_online_retrieval.py +++ b/sdk/python/tests/integration/online_store/test_online_retrieval.py @@ -54,7 +54,7 @@ def test_online() -> None: ) customer_key = EntityKeyProto( - join_keys=["customer"], entity_values=[ValueProto(int64_val=5)] + join_keys=["customer"], entity_values=[ValueProto(string_val="5")] ) provider.online_write_batch( config=store.config, @@ -76,7 +76,7 @@ def test_online() -> None: customer_key = EntityKeyProto( join_keys=["customer", "driver"], - entity_values=[ValueProto(int64_val=5), ValueProto(int64_val=1)], + entity_values=[ValueProto(string_val="5"), ValueProto(int64_val=1)], ) provider.online_write_batch( config=store.config, @@ -100,7 +100,7 @@ def test_online() -> None: "customer_profile:name", "customer_driver_combined:trips", ], - entity_rows=[{"driver": 1, "customer": 5}, {"driver": 1, "customer": 5}], + entity_rows=[{"driver": 1, "customer": "5"}, {"driver": 1, "customer": 5}], full_feature_names=False, ).to_dict() @@ -108,7 +108,7 @@ def test_online() -> None: assert "avg_orders_day" in result assert "name" in result assert result["driver"] == [1, 1] - assert result["customer"] == [5, 5] + assert result["customer"] == ["5", "5"] assert result["lon"] == ["1.0", "1.0"] assert result["avg_orders_day"] == [1.0, 1.0] assert result["name"] == ["John", "John"] @@ -311,7 +311,7 @@ def test_online_to_df(): 6 6.0 foo6 60 """ customer_key = EntityKeyProto( - join_keys=["customer"], entity_values=[ValueProto(int64_val=c)] + join_keys=["customer"], entity_values=[ValueProto(string_val=str(c))] ) provider.online_write_batch( config=store.config, @@ -341,7 +341,7 @@ def test_online_to_df(): """ combo_keys = EntityKeyProto( join_keys=["customer", "driver"], - entity_values=[ValueProto(int64_val=c), ValueProto(int64_val=d)], + entity_values=[ValueProto(string_val=str(c)), ValueProto(int64_val=d)], ) provider.online_write_batch( config=store.config, @@ -382,7 +382,7 @@ def test_online_to_df(): """ df_dict = { "driver": driver_ids, - "customer": customer_ids, + "customer": [str(c) for c in customer_ids], "lon": [str(d * lon_multiply) for d in driver_ids], "lat": [d * lat_multiply for d in driver_ids], "avg_orders_day": [c * avg_order_day_multiply for c in customer_ids], From b4d12bd0444706566c2103c7f53f9efdbec32e2a Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Fri, 7 Jan 2022 10:25:25 -0800 Subject: [PATCH 05/15] Refactor all importer logic to belong in feast.importer (#2199) * Refactor all importer logic to belong in feast.importer Signed-off-by: Felix Wang * Fix unit tests error messages to match feast.errors Signed-off-by: Felix Wang * Rename get_class_from_module to import_class Signed-off-by: Felix Wang * Change class_type checking logic Signed-off-by: Felix Wang * Change error name to FeastInvalidBaseClass Signed-off-by: Felix Wang --- sdk/python/feast/errors.py | 15 +++---- sdk/python/feast/importer.py | 39 ++++++++++++++----- sdk/python/feast/infra/infra_object.py | 4 +- .../infra/offline_stores/offline_utils.py | 29 +++----------- .../feast/infra/online_stores/helpers.py | 28 +++---------- sdk/python/feast/infra/provider.py | 5 ++- sdk/python/feast/registry.py | 6 +-- sdk/python/feast/repo_config.py | 10 ++--- .../integration/registration/test_cli.py | 8 ++-- 9 files changed, 63 insertions(+), 81 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index f6a66bea5a..615069e579 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -103,14 +103,16 @@ def __init__(self, feature_server_type: str): class FeastModuleImportError(Exception): - def __init__(self, module_name: str, module_type: str): - super().__init__(f"Could not import {module_type} module '{module_name}'") + def __init__(self, module_name: str, class_name: str): + super().__init__( + f"Could not import module '{module_name}' while attempting to load class '{class_name}'" + ) class FeastClassImportError(Exception): - def __init__(self, module_name, class_name, class_type="provider"): + def __init__(self, module_name: str, class_name: str): super().__init__( - f"Could not import {class_type} '{class_name}' from module '{module_name}'" + f"Could not import class '{class_name}' from module '{module_name}'" ) @@ -168,11 +170,10 @@ def __init__(self, online_store_class_name: str): ) -class FeastClassInvalidName(Exception): +class FeastInvalidBaseClass(Exception): def __init__(self, class_name: str, class_type: str): super().__init__( - f"Config Class '{class_name}' " - f"should end with the string `{class_type}`.'" + f"Class '{class_name}' should have `{class_type}` as a base class." ) diff --git a/sdk/python/feast/importer.py b/sdk/python/feast/importer.py index 5dcd7c71c1..bbd592101a 100644 --- a/sdk/python/feast/importer.py +++ b/sdk/python/feast/importer.py @@ -1,28 +1,47 @@ import importlib -from feast import errors +from feast.errors import ( + FeastClassImportError, + FeastInvalidBaseClass, + FeastModuleImportError, +) -def get_class_from_type(module_name: str, class_name: str, class_type: str): - if not class_name.endswith(class_type): - raise errors.FeastClassInvalidName(class_name, class_type) +def import_class(module_name: str, class_name: str, class_type: str = None): + """ + Dynamically loads and returns a class from a module. - # Try importing the module that contains the custom provider + Args: + module_name: The name of the module. + class_name: The name of the class. + class_type: Optional name of a base class of the class. + + Raises: + FeastInvalidBaseClass: If the class name does not end with the specified suffix. + FeastModuleImportError: If the module cannot be imported. + FeastClassImportError: If the class cannot be imported. + """ + # Try importing the module. try: module = importlib.import_module(module_name) except Exception as e: # The original exception can be anything - either module not found, # or any other kind of error happening during the module import time. # So we should include the original error as well in the stack trace. - raise errors.FeastModuleImportError(module_name, class_type) from e + raise FeastModuleImportError(module_name, class_name) from e - # Try getting the provider class definition + # Try getting the class. try: _class = getattr(module, class_name) except AttributeError: # This can only be one type of error, when class_name attribute does not exist in the module # So we don't have to include the original exception here - raise errors.FeastClassImportError( - module_name, class_name, class_type=class_type - ) from None + raise FeastClassImportError(module_name, class_name) from None + + # Check if the class is a subclass of the base class. + if class_type and not any( + base_class.__name__ == class_type for base_class in _class.mro() + ): + raise FeastInvalidBaseClass(class_name, class_type) + return _class diff --git a/sdk/python/feast/infra/infra_object.py b/sdk/python/feast/infra/infra_object.py index f1eda19581..3cd00899fe 100644 --- a/sdk/python/feast/infra/infra_object.py +++ b/sdk/python/feast/infra/infra_object.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import Any, List -from feast.importer import get_class_from_type +from feast.importer import import_class from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto @@ -106,4 +106,4 @@ def from_proto(cls, infra_proto: InfraProto): def _get_infra_object_class_from_type(infra_object_class_type: str): module_name, infra_object_class_name = infra_object_class_type.rsplit(".", 1) - return get_class_from_type(module_name, infra_object_class_name, "Object") + return import_class(module_name, infra_object_class_name) diff --git a/sdk/python/feast/infra/offline_stores/offline_utils.py b/sdk/python/feast/infra/offline_stores/offline_utils.py index 6debe14ca0..0b60c3493d 100644 --- a/sdk/python/feast/infra/offline_stores/offline_utils.py +++ b/sdk/python/feast/infra/offline_stores/offline_utils.py @@ -1,4 +1,3 @@ -import importlib import uuid from dataclasses import asdict, dataclass from datetime import datetime, timedelta @@ -12,11 +11,10 @@ import feast from feast.errors import ( EntityTimestampInferenceException, - FeastClassImportError, FeastEntityDFMissingColumnsError, - FeastModuleImportError, ) from feast.feature_view import FeatureView +from feast.importer import import_class from feast.infra.offline_stores.offline_store import OfflineStore from feast.infra.provider import _get_requested_feature_views_to_features_dict from feast.registry import Registry @@ -204,27 +202,10 @@ def get_temp_entity_table_name() -> str: return "feast_entity_df_" + uuid.uuid4().hex -def get_offline_store_from_config(offline_store_config: Any,) -> OfflineStore: - """Get the offline store from offline store config""" - +def get_offline_store_from_config(offline_store_config: Any) -> OfflineStore: + """Creates an offline store corresponding to the given offline store config.""" module_name = offline_store_config.__module__ qualified_name = type(offline_store_config).__name__ - store_class_name = qualified_name.replace("Config", "") - try: - module = importlib.import_module(module_name) - except Exception as e: - # The original exception can be anything - either module not found, - # or any other kind of error happening during the module import time. - # So we should include the original error as well in the stack trace. - raise FeastModuleImportError(module_name, "OfflineStore") from e - - # Try getting the provider class definition - try: - offline_store_class = getattr(module, store_class_name) - except AttributeError: - # This can only be one type of error, when class_name attribute does not exist in the module - # So we don't have to include the original exception here - raise FeastClassImportError( - module_name, store_class_name, class_type="OfflineStore" - ) from None + class_name = qualified_name.replace("Config", "") + offline_store_class = import_class(module_name, class_name, "OfflineStore") return offline_store_class() diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 5e01ddb263..b206c08b7c 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -1,10 +1,9 @@ -import importlib import struct from typing import Any, List import mmh3 -from feast import errors +from feast.importer import import_class from feast.infra.key_encoding_utils import ( serialize_entity_key, serialize_entity_key_prefix, @@ -13,29 +12,12 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -def get_online_store_from_config(online_store_config: Any,) -> OnlineStore: - """Get the online store from online store config""" - +def get_online_store_from_config(online_store_config: Any) -> OnlineStore: + """Creates an online store corresponding to the given online store config.""" module_name = online_store_config.__module__ qualified_name = type(online_store_config).__name__ - store_class_name = qualified_name.replace("Config", "") - try: - module = importlib.import_module(module_name) - except Exception as e: - # The original exception can be anything - either module not found, - # or any other kind of error happening during the module import time. - # So we should include the original error as well in the stack trace. - raise errors.FeastModuleImportError(module_name, "OnlineStore") from e - - # Try getting the provider class definition - try: - online_store_class = getattr(module, store_class_name) - except AttributeError: - # This can only be one type of error, when class_name attribute does not exist in the module - # So we don't have to include the original exception here - raise errors.FeastClassImportError( - module_name, store_class_name, class_type="OnlineStore" - ) from None + class_name = qualified_name.replace("Config", "") + online_store_class = import_class(module_name, class_name, "OnlineStore") return online_store_class() diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 32bca8f7d7..3c761f1195 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -8,9 +8,10 @@ import pyarrow from tqdm import tqdm -from feast import errors, importer +from feast import errors from feast.entity import Entity from feast.feature_view import DUMMY_ENTITY_ID, FeatureView +from feast.importer import import_class from feast.infra.offline_stores.offline_store import RetrievalJob from feast.on_demand_feature_view import OnDemandFeatureView from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto @@ -172,7 +173,7 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: # For example, provider 'foo.bar.MyProvider' will be parsed into 'foo.bar' and 'MyProvider' module_name, class_name = provider.rsplit(".", 1) - cls = importer.get_class_from_type(module_name, class_name, "Provider") + cls = import_class(module_name, class_name, "Provider") return cls(config) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index d5c4c08048..0c058a0d46 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -23,7 +23,6 @@ from google.protobuf.json_format import MessageToDict from proto import Message -from feast import importer from feast.base_feature_view import BaseFeatureView from feast.diff.FcoDiff import ( FcoDiff, @@ -42,6 +41,7 @@ ) from feast.feature_service import FeatureService from feast.feature_view import FeatureView +from feast.importer import import_class from feast.infra.infra_object import Infra from feast.on_demand_feature_view import OnDemandFeatureView from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto @@ -75,9 +75,7 @@ def get_registry_store_class_from_type(registry_store_type: str): registry_store_type = REGISTRY_STORE_CLASS_FOR_TYPE[registry_store_type] module_name, registry_store_class_name = registry_store_type.rsplit(".", 1) - return importer.get_class_from_type( - module_name, registry_store_class_name, "RegistryStore" - ) + return import_class(module_name, registry_store_class_name, "RegistryStore") def get_registry_store_class_from_scheme(registry_path: str): diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 70e64c845c..26309fe9d7 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -20,7 +20,7 @@ FeastFeatureServerTypeSetError, FeastProviderNotSetError, ) -from feast.importer import get_class_from_type +from feast.importer import import_class from feast.usage import log_exceptions # These dict exists so that: @@ -302,7 +302,7 @@ def __repr__(self) -> str: def get_data_source_class_from_type(data_source_type: str): module_name, config_class_name = data_source_type.rsplit(".", 1) - return get_class_from_type(module_name, config_class_name, "Source") + return import_class(module_name, config_class_name, "DataSource") def get_online_config_from_type(online_store_type: str): @@ -313,7 +313,7 @@ def get_online_config_from_type(online_store_type: str): module_name, online_store_class_type = online_store_type.rsplit(".", 1) config_class_name = f"{online_store_class_type}Config" - return get_class_from_type(module_name, config_class_name, config_class_name) + return import_class(module_name, config_class_name, config_class_name) def get_offline_config_from_type(offline_store_type: str): @@ -324,7 +324,7 @@ def get_offline_config_from_type(offline_store_type: str): module_name, offline_store_class_type = offline_store_type.rsplit(".", 1) config_class_name = f"{offline_store_class_type}Config" - return get_class_from_type(module_name, config_class_name, config_class_name) + return import_class(module_name, config_class_name, config_class_name) def get_feature_server_config_from_type(feature_server_type: str): @@ -334,7 +334,7 @@ def get_feature_server_config_from_type(feature_server_type: str): feature_server_type = FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE[feature_server_type] module_name, config_class_name = feature_server_type.rsplit(".", 1) - return get_class_from_type(module_name, config_class_name, config_class_name) + return import_class(module_name, config_class_name, config_class_name) def load_repo_config(repo_path: Path) -> RepoConfig: diff --git a/sdk/python/tests/integration/registration/test_cli.py b/sdk/python/tests/integration/registration/test_cli.py index f05674ea5c..0fe73316ad 100644 --- a/sdk/python/tests/integration/registration/test_cli.py +++ b/sdk/python/tests/integration/registration/test_cli.py @@ -211,14 +211,14 @@ def test_3rd_party_providers() -> None: return_code, output = runner.run_with_output(["apply"], cwd=repo_path) assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains( - b"Could not import Provider module 'feast_foo'" + b"Could not import module 'feast_foo' while attempting to load class 'Provider'" ) # Check with incorrect third-party provider name (with dots) with setup_third_party_provider_repo("foo.FooProvider") as repo_path: return_code, output = runner.run_with_output(["apply"], cwd=repo_path) assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains( - b"Could not import Provider 'FooProvider' from module 'foo'" + b"Could not import class 'FooProvider' from module 'foo'" ) # Check with correct third-party provider name with setup_third_party_provider_repo("foo.provider.FooProvider") as repo_path: @@ -243,14 +243,14 @@ def test_3rd_party_registry_store() -> None: return_code, output = runner.run_with_output(["apply"], cwd=repo_path) assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains( - b"Could not import RegistryStore module 'feast_foo'" + b"Could not import module 'feast_foo' while attempting to load class 'RegistryStore'" ) # Check with incorrect third-party registry store name (with dots) with setup_third_party_registry_store_repo("foo.FooRegistryStore") as repo_path: return_code, output = runner.run_with_output(["apply"], cwd=repo_path) assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains( - b"Could not import RegistryStore 'FooRegistryStore' from module 'foo'" + b"Could not import class 'FooRegistryStore' from module 'foo'" ) # Check with correct third-party registry store name with setup_third_party_registry_store_repo( From a16c5e475f2f86d4252aaaa7378befb89ffb81c2 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 7 Jan 2022 15:24:52 -0600 Subject: [PATCH 06/15] Modify issue templates to automatically attach labels (#2205) Signed-off-by: Danny Chiao --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index c405c1f084..9263376d7e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Bug report about: Create a report to help us improve title: '' -labels: '' +labels: 'kind/bug' assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d61..d73d644481 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: '' +labels: 'kind/feature' assignees: '' --- From 36c1d4602969b1e1c5e9e67cf180d6bbf026d9f1 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 7 Jan 2022 16:33:52 -0600 Subject: [PATCH 07/15] Add default priority for bug reports (#2207) Signed-off-by: Danny Chiao --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 9263376d7e..2f2d0d2f5e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Bug report about: Create a report to help us improve title: '' -labels: 'kind/bug' +labels: 'kind/bug, priority/p2' assignees: '' --- From d241a8451a3799d15297365b7bee7f3c571d2d13 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Mon, 10 Jan 2022 06:27:11 -0800 Subject: [PATCH 08/15] Implement diff_infra_protos method for feast plan (#2204) * Rename proto methods for SqliteTable, DynamoDBTable, DatastoreTable Signed-off-by: Felix Wang * Implement diff_infra Signed-off-by: Felix Wang * Add tests for diff_infra logic Signed-off-by: Felix Wang * Fix Datastore infra object Signed-off-by: Felix Wang --- sdk/python/feast/diff/infra_diff.py | 136 +++++++++++++++- sdk/python/feast/infra/infra_object.py | 17 +- .../feast/infra/online_stores/datastore.py | 49 +++--- .../feast/infra/online_stores/dynamodb.py | 19 ++- .../feast/infra/online_stores/sqlite.py | 19 ++- sdk/python/tests/unit/diff/test_infra_diff.py | 154 ++++++++++++++++++ 6 files changed, 347 insertions(+), 47 deletions(-) create mode 100644 sdk/python/tests/unit/diff/test_infra_diff.py diff --git a/sdk/python/feast/diff/infra_diff.py b/sdk/python/feast/diff/infra_diff.py index 458b7e1e01..d716422261 100644 --- a/sdk/python/feast/diff/infra_diff.py +++ b/sdk/python/feast/diff/infra_diff.py @@ -1,16 +1,30 @@ from dataclasses import dataclass -from typing import Any, List +from typing import Any, Iterable, List, Tuple, TypeVar from feast.diff.property_diff import PropertyDiff, TransitionType +from feast.infra.infra_object import ( + DATASTORE_INFRA_OBJECT_CLASS_TYPE, + DYNAMODB_INFRA_OBJECT_CLASS_TYPE, + SQLITE_INFRA_OBJECT_CLASS_TYPE, + InfraObject, +) +from feast.protos.feast.core.DatastoreTable_pb2 import ( + DatastoreTable as DatastoreTableProto, +) +from feast.protos.feast.core.DynamoDBTable_pb2 import ( + DynamoDBTable as DynamoDBTableProto, +) +from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto +from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto @dataclass class InfraObjectDiff: name: str infra_object_type: str - current_fco: Any - new_fco: Any - fco_property_diffs: List[PropertyDiff] + current_infra_object: Any + new_infra_object: Any + infra_object_property_diffs: List[PropertyDiff] transition_type: TransitionType @@ -26,3 +40,117 @@ def update(self): def to_string(self): pass + + +U = TypeVar("U", DatastoreTableProto, DynamoDBTableProto, SqliteTableProto) + + +def tag_infra_proto_objects_for_keep_delete_add( + existing_objs: Iterable[U], desired_objs: Iterable[U] +) -> Tuple[Iterable[U], Iterable[U], Iterable[U]]: + existing_obj_names = {e.name for e in existing_objs} + desired_obj_names = {e.name for e in desired_objs} + + objs_to_add = [e for e in desired_objs if e.name not in existing_obj_names] + objs_to_keep = [e for e in desired_objs if e.name in existing_obj_names] + objs_to_delete = [e for e in existing_objs if e.name not in desired_obj_names] + + return objs_to_keep, objs_to_delete, objs_to_add + + +def diff_infra_protos( + current_infra_proto: InfraProto, new_infra_proto: InfraProto +) -> InfraDiff: + infra_diff = InfraDiff() + + infra_object_class_types_to_str = { + DATASTORE_INFRA_OBJECT_CLASS_TYPE: "datastore table", + DYNAMODB_INFRA_OBJECT_CLASS_TYPE: "dynamodb table", + SQLITE_INFRA_OBJECT_CLASS_TYPE: "sqlite table", + } + + for infra_object_class_type in infra_object_class_types_to_str: + current_infra_objects = get_infra_object_protos_by_type( + current_infra_proto, infra_object_class_type + ) + new_infra_objects = get_infra_object_protos_by_type( + new_infra_proto, infra_object_class_type + ) + ( + infra_objects_to_keep, + infra_objects_to_delete, + infra_objects_to_add, + ) = tag_infra_proto_objects_for_keep_delete_add( + current_infra_objects, new_infra_objects, + ) + + for e in infra_objects_to_add: + infra_diff.infra_object_diffs.append( + InfraObjectDiff( + e.name, + infra_object_class_types_to_str[infra_object_class_type], + None, + e, + [], + TransitionType.CREATE, + ) + ) + for e in infra_objects_to_delete: + infra_diff.infra_object_diffs.append( + InfraObjectDiff( + e.name, + infra_object_class_types_to_str[infra_object_class_type], + e, + None, + [], + TransitionType.DELETE, + ) + ) + for e in infra_objects_to_keep: + current_infra_object = [ + _e for _e in current_infra_objects if _e.name == e.name + ][0] + infra_diff.infra_object_diffs.append( + diff_between( + current_infra_object, + e, + infra_object_class_types_to_str[infra_object_class_type], + ) + ) + + return infra_diff + + +def get_infra_object_protos_by_type( + infra_proto: InfraProto, infra_object_class_type: str +) -> List[U]: + return [ + InfraObject.from_infra_object_proto(infra_object).to_proto() + for infra_object in infra_proto.infra_objects + if infra_object.infra_object_class_type == infra_object_class_type + ] + + +FIELDS_TO_IGNORE = {"project"} + + +def diff_between(current: U, new: U, infra_object_type: str) -> InfraObjectDiff: + assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name + property_diffs = [] + transition: TransitionType = TransitionType.UNCHANGED + if current != new: + for _field in current.DESCRIPTOR.fields: + if _field.name in FIELDS_TO_IGNORE: + continue + if getattr(current, _field.name) != getattr(new, _field.name): + transition = TransitionType.UPDATE + property_diffs.append( + PropertyDiff( + _field.name, + getattr(current, _field.name), + getattr(new, _field.name), + ) + ) + return InfraObjectDiff( + new.name, infra_object_type, current, new, property_diffs, transition, + ) diff --git a/sdk/python/feast/infra/infra_object.py b/sdk/python/feast/infra/infra_object.py index 3cd00899fe..282b4bcfab 100644 --- a/sdk/python/feast/infra/infra_object.py +++ b/sdk/python/feast/infra/infra_object.py @@ -19,6 +19,10 @@ from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto +DATASTORE_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_stores.datastore.DatastoreTable" +DYNAMODB_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_stores.dynamodb.DynamoDBTable" +SQLITE_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_store.sqlite.SqliteTable" + class InfraObject(ABC): """ @@ -26,13 +30,18 @@ class InfraObject(ABC): """ @abstractmethod - def to_proto(self) -> InfraObjectProto: + def to_infra_object_proto(self) -> InfraObjectProto: + """Converts an InfraObject to its protobuf representation, wrapped in an InfraObjectProto.""" + pass + + @abstractmethod + def to_proto(self) -> Any: """Converts an InfraObject to its protobuf representation.""" pass @staticmethod @abstractmethod - def from_proto(infra_object_proto: InfraObjectProto) -> Any: + def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: """ Returns an InfraObject created from a protobuf representation. @@ -46,7 +55,7 @@ def from_proto(infra_object_proto: InfraObjectProto) -> Any: cls = _get_infra_object_class_from_type( infra_object_proto.infra_object_class_type ) - return cls.from_proto(infra_object_proto) + return cls.from_infra_object_proto(infra_object_proto) raise ValueError("Could not identify the type of the InfraObject.") @@ -97,7 +106,7 @@ def from_proto(cls, infra_proto: InfraProto): """ infra = cls() cls.infra_objects += [ - InfraObject.from_proto(infra_object_proto) + InfraObject.from_infra_object_proto(infra_object_proto) for infra_object_proto in infra_proto.infra_objects ] diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index f788f1bc74..348583a202 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -23,8 +23,9 @@ from pydantic.typing import Literal from feast import Entity, utils +from feast.errors import FeastProviderLoginError from feast.feature_view import FeatureView -from feast.infra.infra_object import InfraObject +from feast.infra.infra_object import DATASTORE_INFRA_OBJECT_CLASS_TYPE, InfraObject from feast.infra.online_stores.helpers import compute_entity_id from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.core.DatastoreTable_pb2 import ( @@ -43,7 +44,7 @@ from google.cloud import datastore from google.cloud.datastore.client import Key except ImportError as e: - from feast.errors import FeastExtrasDependencyImportError, FeastProviderLoginError + from feast.errors import FeastExtrasDependencyImportError raise FeastExtrasDependencyImportError("gcp", str(e)) @@ -332,14 +333,12 @@ class DatastoreTable(InfraObject): name: The name of the table. project_id (optional): The GCP project id. namespace (optional): Datastore namespace. - client: Datastore client. """ project: str name: str project_id: Optional[str] namespace: Optional[str] - client: datastore.Client def __init__( self, @@ -352,24 +351,26 @@ def __init__( self.name = name self.project_id = project_id self.namespace = namespace - self.client = _initialize_client(self.project_id, self.namespace) - def to_proto(self) -> InfraObjectProto: + def to_infra_object_proto(self) -> InfraObjectProto: + datastore_table_proto = self.to_proto() + return InfraObjectProto( + infra_object_class_type=DATASTORE_INFRA_OBJECT_CLASS_TYPE, + datastore_table=datastore_table_proto, + ) + + def to_proto(self) -> Any: datastore_table_proto = DatastoreTableProto() datastore_table_proto.project = self.project datastore_table_proto.name = self.name if self.project_id: - datastore_table_proto.project_id.FromString(bytes(self.project_id, "utf-8")) + datastore_table_proto.project_id.value = self.project_id if self.namespace: - datastore_table_proto.namespace.FromString(bytes(self.namespace, "utf-8")) - - return InfraObjectProto( - infra_object_class_type="feast.infra.online_stores.datastore.DatastoreTable", - datastore_table=datastore_table_proto, - ) + datastore_table_proto.namespace.value = self.namespace + return datastore_table_proto @staticmethod - def from_proto(infra_object_proto: InfraObjectProto) -> Any: + def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: datastore_table = DatastoreTable( project=infra_object_proto.datastore_table.project, name=infra_object_proto.datastore_table.name, @@ -377,26 +378,28 @@ def from_proto(infra_object_proto: InfraObjectProto) -> Any: if infra_object_proto.datastore_table.HasField("project_id"): datastore_table.project_id = ( - infra_object_proto.datastore_table.project_id.SerializeToString() - ).decode("utf-8") + infra_object_proto.datastore_table.project_id.value + ) if infra_object_proto.datastore_table.HasField("namespace"): datastore_table.namespace = ( - infra_object_proto.datastore_table.namespace.SerializeToString() - ).decode("utf-8") + infra_object_proto.datastore_table.namespace.value + ) return datastore_table def update(self): - key = self.client.key("Project", self.project, "Table", self.name) + client = _initialize_client(self.project_id, self.namespace) + key = client.key("Project", self.project, "Table", self.name) entity = datastore.Entity( key=key, exclude_from_indexes=("created_ts", "event_ts", "values") ) entity.update({"created_ts": datetime.utcnow()}) - self.client.put(entity) + client.put(entity) def teardown(self): - key = self.client.key("Project", self.project, "Table", self.name) - _delete_all_values(self.client, key) + client = _initialize_client(self.project_id, self.namespace) + key = client.key("Project", self.project, "Table", self.name) + _delete_all_values(client, key) # Delete the table metadata datastore entity - self.client.delete(key) + client.delete(key) diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 377e10c308..202cfa54bb 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -19,7 +19,7 @@ from pydantic.typing import Literal from feast import Entity, FeatureView, utils -from feast.infra.infra_object import InfraObject +from feast.infra.infra_object import DYNAMODB_INFRA_OBJECT_CLASS_TYPE, InfraObject from feast.infra.online_stores.helpers import compute_entity_id from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.core.DynamoDBTable_pb2 import ( @@ -234,18 +234,21 @@ def __init__(self, name: str, region: str): self.name = name self.region = region - def to_proto(self) -> InfraObjectProto: - dynamodb_table_proto = DynamoDBTableProto() - dynamodb_table_proto.name = self.name - dynamodb_table_proto.region = self.region - + def to_infra_object_proto(self) -> InfraObjectProto: + dynamodb_table_proto = self.to_proto() return InfraObjectProto( - infra_object_class_type="feast.infra.online_stores.dynamodb.DynamoDBTable", + infra_object_class_type=DYNAMODB_INFRA_OBJECT_CLASS_TYPE, dynamodb_table=dynamodb_table_proto, ) + def to_proto(self) -> Any: + dynamodb_table_proto = DynamoDBTableProto() + dynamodb_table_proto.name = self.name + dynamodb_table_proto.region = self.region + return dynamodb_table_proto + @staticmethod - def from_proto(infra_object_proto: InfraObjectProto) -> Any: + def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: return DynamoDBTable( name=infra_object_proto.dynamodb_table.name, region=infra_object_proto.dynamodb_table.region, diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 206e2eb0d5..2dcbf319c3 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -23,7 +23,7 @@ from feast import Entity from feast.feature_view import FeatureView -from feast.infra.infra_object import InfraObject +from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject from feast.infra.key_encoding_utils import serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto @@ -241,18 +241,21 @@ def __init__(self, path: str, name: str): self.name = name self.conn = _initialize_conn(path) - def to_proto(self) -> InfraObjectProto: - sqlite_table_proto = SqliteTableProto() - sqlite_table_proto.path = self.path - sqlite_table_proto.name = self.name - + def to_infra_object_proto(self) -> InfraObjectProto: + sqlite_table_proto = self.to_proto() return InfraObjectProto( - infra_object_class_type="feast.infra.online_store.sqlite.SqliteTable", + infra_object_class_type=SQLITE_INFRA_OBJECT_CLASS_TYPE, sqlite_table=sqlite_table_proto, ) + def to_proto(self) -> Any: + sqlite_table_proto = SqliteTableProto() + sqlite_table_proto.path = self.path + sqlite_table_proto.name = self.name + return sqlite_table_proto + @staticmethod - def from_proto(infra_object_proto: InfraObjectProto) -> Any: + def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: return SqliteTable( path=infra_object_proto.sqlite_table.path, name=infra_object_proto.sqlite_table.name, diff --git a/sdk/python/tests/unit/diff/test_infra_diff.py b/sdk/python/tests/unit/diff/test_infra_diff.py new file mode 100644 index 0000000000..8e3d5b765f --- /dev/null +++ b/sdk/python/tests/unit/diff/test_infra_diff.py @@ -0,0 +1,154 @@ +from google.protobuf import wrappers_pb2 as wrappers + +from feast.diff.infra_diff import ( + diff_between, + diff_infra_protos, + tag_infra_proto_objects_for_keep_delete_add, +) +from feast.diff.property_diff import TransitionType +from feast.infra.online_stores.datastore import DatastoreTable +from feast.infra.online_stores.dynamodb import DynamoDBTable +from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto + + +def test_tag_infra_proto_objects_for_keep_delete_add(): + to_delete = DynamoDBTable(name="to_delete", region="us-west-2").to_proto() + to_add = DynamoDBTable(name="to_add", region="us-west-2").to_proto() + unchanged_table = DynamoDBTable(name="unchanged", region="us-west-2").to_proto() + pre_changed = DynamoDBTable(name="table", region="us-west-2").to_proto() + post_changed = DynamoDBTable(name="table", region="us-east-2").to_proto() + + keep, delete, add = tag_infra_proto_objects_for_keep_delete_add( + [to_delete, unchanged_table, pre_changed], + [to_add, unchanged_table, post_changed], + ) + + assert len(list(keep)) == 2 + assert unchanged_table in keep + assert post_changed in keep + assert to_add not in keep + assert len(list(delete)) == 1 + assert to_delete in delete + assert unchanged_table not in delete + assert pre_changed not in delete + assert len(list(add)) == 1 + assert to_add in add + assert unchanged_table not in add + assert post_changed not in add + + +def test_diff_between_datastore_tables(): + pre_changed = DatastoreTable( + project="test", name="table", project_id="pre", namespace="pre" + ).to_proto() + post_changed = DatastoreTable( + project="test", name="table", project_id="post", namespace="post" + ).to_proto() + + infra_object_diff = diff_between(pre_changed, pre_changed, "datastore table") + infra_object_property_diffs = infra_object_diff.infra_object_property_diffs + assert len(infra_object_property_diffs) == 0 + + infra_object_diff = diff_between(pre_changed, post_changed, "datastore table") + infra_object_property_diffs = infra_object_diff.infra_object_property_diffs + assert len(infra_object_property_diffs) == 2 + + assert infra_object_property_diffs[0].property_name == "project_id" + assert infra_object_property_diffs[0].val_existing == wrappers.StringValue( + value="pre" + ) + assert infra_object_property_diffs[0].val_declared == wrappers.StringValue( + value="post" + ) + assert infra_object_property_diffs[1].property_name == "namespace" + assert infra_object_property_diffs[1].val_existing == wrappers.StringValue( + value="pre" + ) + assert infra_object_property_diffs[1].val_declared == wrappers.StringValue( + value="post" + ) + + +def test_diff_infra_protos(): + to_delete = DynamoDBTable(name="to_delete", region="us-west-2") + to_add = DynamoDBTable(name="to_add", region="us-west-2") + unchanged_table = DynamoDBTable(name="unchanged", region="us-west-2") + pre_changed = DatastoreTable( + project="test", name="table", project_id="pre", namespace="pre" + ) + post_changed = DatastoreTable( + project="test", name="table", project_id="post", namespace="post" + ) + + infra_objects_before = [to_delete, unchanged_table, pre_changed] + infra_objects_after = [to_add, unchanged_table, post_changed] + + infra_proto_before = InfraProto() + infra_proto_before.infra_objects.extend( + [obj.to_infra_object_proto() for obj in infra_objects_before] + ) + + infra_proto_after = InfraProto() + infra_proto_after.infra_objects.extend( + [obj.to_infra_object_proto() for obj in infra_objects_after] + ) + + infra_diff = diff_infra_protos(infra_proto_before, infra_proto_after) + infra_object_diffs = infra_diff.infra_object_diffs + + # There should be one addition, one deletion, one unchanged, and one changed. + assert len(infra_object_diffs) == 4 + + additions = [ + infra_object_diff + for infra_object_diff in infra_object_diffs + if infra_object_diff.transition_type == TransitionType.CREATE + ] + assert len(additions) == 1 + assert not additions[0].current_infra_object + assert additions[0].new_infra_object == to_add.to_proto() + assert len(additions[0].infra_object_property_diffs) == 0 + + deletions = [ + infra_object_diff + for infra_object_diff in infra_object_diffs + if infra_object_diff.transition_type == TransitionType.DELETE + ] + assert len(deletions) == 1 + assert deletions[0].current_infra_object == to_delete.to_proto() + assert not deletions[0].new_infra_object + assert len(deletions[0].infra_object_property_diffs) == 0 + + unchanged = [ + infra_object_diff + for infra_object_diff in infra_object_diffs + if infra_object_diff.transition_type == TransitionType.UNCHANGED + ] + assert len(unchanged) == 1 + assert unchanged[0].current_infra_object == unchanged_table.to_proto() + assert unchanged[0].new_infra_object == unchanged_table.to_proto() + assert len(unchanged[0].infra_object_property_diffs) == 0 + + updates = [ + infra_object_diff + for infra_object_diff in infra_object_diffs + if infra_object_diff.transition_type == TransitionType.UPDATE + ] + assert len(updates) == 1 + assert updates[0].current_infra_object == pre_changed.to_proto() + assert updates[0].new_infra_object == post_changed.to_proto() + assert len(updates[0].infra_object_property_diffs) == 2 + assert updates[0].infra_object_property_diffs[0].property_name == "project_id" + assert updates[0].infra_object_property_diffs[ + 0 + ].val_existing == wrappers.StringValue(value="pre") + assert updates[0].infra_object_property_diffs[ + 0 + ].val_declared == wrappers.StringValue(value="post") + assert updates[0].infra_object_property_diffs[1].property_name == "namespace" + assert updates[0].infra_object_property_diffs[ + 1 + ].val_existing == wrappers.StringValue(value="pre") + assert updates[0].infra_object_property_diffs[ + 1 + ].val_declared == wrappers.StringValue(value="post") From 97dd41e0c0e1a9e8050f3931eb610f59789edeed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 06:42:12 -0800 Subject: [PATCH 09/15] Bump protobuf-java from 3.12.2 to 3.16.1 in /java (#2208) Bumps [protobuf-java](https://github.com/protocolbuffers/protobuf) from 3.12.2 to 3.16.1. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.12.2...v3.16.1) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 38f037431c..6857bce4c0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -46,7 +46,7 @@ 1.30.2 3.12.2 - 3.12.2 + 3.16.1 2.3.1.RELEASE 5.2.7.RELEASE 5.3.0.RELEASE From d5cb0440e4f8df76310aed026d6c90acc0a19fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diogo=20Magalh=C3=A3es=20Martins?= Date: Tue, 11 Jan 2022 01:27:35 -0300 Subject: [PATCH 10/15] Updates to click==8.* (#2210) * Updates to click==8.* Signed-off-by: diogommartins * Updates pip tools requirements to click==8.0.3 Signed-off-by: diogommartins --- sdk/python/requirements/py3.7-ci-requirements.txt | 2 +- sdk/python/requirements/py3.7-requirements.txt | 2 +- sdk/python/requirements/py3.8-ci-requirements.txt | 2 +- sdk/python/requirements/py3.8-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- sdk/python/setup.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 017652873a..a4fd01a47a 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -85,7 +85,7 @@ charset-normalizer==2.0.8 # via # aiohttp # requests -click==7.1.2 +click==8.0.3 # via # black # feast (setup.py) diff --git a/sdk/python/requirements/py3.7-requirements.txt b/sdk/python/requirements/py3.7-requirements.txt index b2473f1c70..4e7408f86e 100644 --- a/sdk/python/requirements/py3.7-requirements.txt +++ b/sdk/python/requirements/py3.7-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.8 # via requests -click==7.1.2 +click==8.0.3 # via # feast (setup.py) # uvicorn diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index a2df153c01..1c57df69b6 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -83,7 +83,7 @@ charset-normalizer==2.0.8 # via # aiohttp # requests -click==7.1.2 +click==8.0.3 # via # black # feast (setup.py) diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index e6887dea55..11b2dbc6ac 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.8 # via requests -click==7.1.2 +click==8.0.3 # via # feast (setup.py) # uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index aa4cee54e4..b8d33ceb85 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -83,7 +83,7 @@ charset-normalizer==2.0.8 # via # aiohttp # requests -click==7.1.2 +click==8.0.3 # via # black # feast (setup.py) diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 4cb45fd809..ce597b8eb1 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.8 # via requests -click==7.1.2 +click==8.0.3 # via # feast (setup.py) # uvicorn diff --git a/sdk/python/setup.py b/sdk/python/setup.py index e797a1216c..ae8d167ff0 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -40,7 +40,7 @@ REQUIRES_PYTHON = ">=3.7.0" REQUIRED = [ - "Click==7.*", + "Click==8.*", "colorama>=0.3.9", "dill==0.3.*", "fastavro>=1.1.0", From 2a95629e8c4a760b282da3ccce0897d6b9c528a0 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Tue, 11 Jan 2022 13:50:51 -0800 Subject: [PATCH 11/15] Modify feature_store.plan to produce an InfraDiff (#2211) * Implement InfraObject.from_proto for easier conversion Signed-off-by: Felix Wang * Implement InfraDiff.update Signed-off-by: Felix Wang * Modify feature_store.plan to produce an InfraDiff Signed-off-by: Felix Wang * Stricter typing for FcoDiff and InfraObjectDiff Signed-off-by: Felix Wang * Small fixes Signed-off-by: Felix Wang * Fix typevar names Signed-off-by: Felix Wang * Add comment Signed-off-by: Felix Wang * Fix protos Signed-off-by: Felix Wang --- sdk/python/feast/diff/FcoDiff.py | 41 +++++++++++------ sdk/python/feast/diff/infra_diff.py | 46 ++++++++++++++----- sdk/python/feast/errors.py | 5 ++ sdk/python/feast/feature_store.py | 25 ++++++++-- sdk/python/feast/infra/infra_object.py | 39 ++++++++++++++-- sdk/python/feast/infra/local.py | 19 +++++--- .../feast/infra/online_stores/datastore.py | 15 ++++++ .../feast/infra/online_stores/dynamodb.py | 6 +++ .../feast/infra/online_stores/online_store.py | 14 ++++++ .../feast/infra/online_stores/sqlite.py | 20 ++++++++ sdk/python/feast/infra/provider.py | 14 ++++++ sdk/python/feast/repo_operations.py | 8 ++-- 12 files changed, 208 insertions(+), 44 deletions(-) diff --git a/sdk/python/feast/diff/FcoDiff.py b/sdk/python/feast/diff/FcoDiff.py index e4b044dcc4..b85897019f 100644 --- a/sdk/python/feast/diff/FcoDiff.py +++ b/sdk/python/feast/diff/FcoDiff.py @@ -1,20 +1,38 @@ from dataclasses import dataclass -from typing import Any, Iterable, List, Set, Tuple, TypeVar +from typing import Generic, Iterable, List, Set, Tuple, TypeVar from feast.base_feature_view import BaseFeatureView from feast.diff.property_diff import PropertyDiff, TransitionType from feast.entity import Entity from feast.feature_service import FeatureService from feast.protos.feast.core.Entity_pb2 import Entity as EntityProto +from feast.protos.feast.core.FeatureService_pb2 import ( + FeatureService as FeatureServiceProto, +) from feast.protos.feast.core.FeatureView_pb2 import FeatureView as FeatureViewProto +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + OnDemandFeatureView as OnDemandFeatureViewProto, +) +from feast.protos.feast.core.RequestFeatureView_pb2 import ( + RequestFeatureView as RequestFeatureViewProto, +) + +FcoProto = TypeVar( + "FcoProto", + EntityProto, + FeatureViewProto, + FeatureServiceProto, + OnDemandFeatureViewProto, + RequestFeatureViewProto, +) @dataclass -class FcoDiff: +class FcoDiff(Generic[FcoProto]): name: str fco_type: str - current_fco: Any - new_fco: Any + current_fco: FcoProto + new_fco: FcoProto fco_property_diffs: List[PropertyDiff] transition_type: TransitionType @@ -30,12 +48,12 @@ def add_fco_diff(self, fco_diff: FcoDiff): self.fco_diffs.append(fco_diff) -T = TypeVar("T", Entity, BaseFeatureView, FeatureService) +Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService) def tag_objects_for_keep_delete_add( - existing_objs: Iterable[T], desired_objs: Iterable[T] -) -> Tuple[Set[T], Set[T], Set[T]]: + existing_objs: Iterable[Fco], desired_objs: Iterable[Fco] +) -> Tuple[Set[Fco], Set[Fco], Set[Fco]]: existing_obj_names = {e.name for e in existing_objs} desired_obj_names = {e.name for e in desired_objs} @@ -46,12 +64,9 @@ def tag_objects_for_keep_delete_add( return objs_to_keep, objs_to_delete, objs_to_add -U = TypeVar("U", EntityProto, FeatureViewProto) - - def tag_proto_objects_for_keep_delete_add( - existing_objs: Iterable[U], desired_objs: Iterable[U] -) -> Tuple[Iterable[U], Iterable[U], Iterable[U]]: + existing_objs: Iterable[FcoProto], desired_objs: Iterable[FcoProto] +) -> Tuple[Iterable[FcoProto], Iterable[FcoProto], Iterable[FcoProto]]: existing_obj_names = {e.spec.name for e in existing_objs} desired_obj_names = {e.spec.name for e in desired_objs} @@ -65,7 +80,7 @@ def tag_proto_objects_for_keep_delete_add( FIELDS_TO_IGNORE = {"project"} -def diff_between(current: U, new: U, object_type: str) -> FcoDiff: +def diff_between(current: FcoProto, new: FcoProto, object_type: str) -> FcoDiff: assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name property_diffs = [] transition: TransitionType = TransitionType.UNCHANGED diff --git a/sdk/python/feast/diff/infra_diff.py b/sdk/python/feast/diff/infra_diff.py index d716422261..fc79a74f67 100644 --- a/sdk/python/feast/diff/infra_diff.py +++ b/sdk/python/feast/diff/infra_diff.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Iterable, List, Tuple, TypeVar +from typing import Generic, Iterable, List, Tuple, TypeVar from feast.diff.property_diff import PropertyDiff, TransitionType from feast.infra.infra_object import ( @@ -17,13 +17,17 @@ from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto +InfraObjectProto = TypeVar( + "InfraObjectProto", DatastoreTableProto, DynamoDBTableProto, SqliteTableProto +) + @dataclass -class InfraObjectDiff: +class InfraObjectDiff(Generic[InfraObjectProto]): name: str infra_object_type: str - current_infra_object: Any - new_infra_object: Any + current_infra_object: InfraObjectProto + new_infra_object: InfraObjectProto infra_object_property_diffs: List[PropertyDiff] transition_type: TransitionType @@ -36,18 +40,34 @@ def __init__(self): self.infra_object_diffs = [] def update(self): - pass + """Apply the infrastructure changes specified in this object.""" + for infra_object_diff in self.infra_object_diffs: + if infra_object_diff.transition_type in [ + TransitionType.DELETE, + TransitionType.UPDATE, + ]: + infra_object = InfraObject.from_proto( + infra_object_diff.current_infra_object + ) + infra_object.teardown() + elif infra_object_diff.transition_type in [ + TransitionType.CREATE, + TransitionType.UPDATE, + ]: + infra_object = InfraObject.from_proto( + infra_object_diff.new_infra_object + ) + infra_object.update() def to_string(self): pass -U = TypeVar("U", DatastoreTableProto, DynamoDBTableProto, SqliteTableProto) - - def tag_infra_proto_objects_for_keep_delete_add( - existing_objs: Iterable[U], desired_objs: Iterable[U] -) -> Tuple[Iterable[U], Iterable[U], Iterable[U]]: + existing_objs: Iterable[InfraObjectProto], desired_objs: Iterable[InfraObjectProto] +) -> Tuple[ + Iterable[InfraObjectProto], Iterable[InfraObjectProto], Iterable[InfraObjectProto] +]: existing_obj_names = {e.name for e in existing_objs} desired_obj_names = {e.name for e in desired_objs} @@ -123,7 +143,7 @@ def diff_infra_protos( def get_infra_object_protos_by_type( infra_proto: InfraProto, infra_object_class_type: str -) -> List[U]: +) -> List[InfraObjectProto]: return [ InfraObject.from_infra_object_proto(infra_object).to_proto() for infra_object in infra_proto.infra_objects @@ -134,7 +154,9 @@ def get_infra_object_protos_by_type( FIELDS_TO_IGNORE = {"project"} -def diff_between(current: U, new: U, infra_object_type: str) -> InfraObjectDiff: +def diff_between( + current: InfraObjectProto, new: InfraObjectProto, infra_object_type: str +) -> InfraObjectDiff: assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name property_diffs = [] transition: TransitionType = TransitionType.UNCHANGED diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 615069e579..8592960acd 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -293,3 +293,8 @@ def __init__(self, actual_class: str, expected_class: str): super().__init__( f"The registry store class was expected to be {expected_class}, but was instead {actual_class}." ) + + +class FeastInvalidInfraObjectType(Exception): + def __init__(self): + super().__init__("Could not identify the type of the InfraObject.") diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f1fee70336..64bf23ebde 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -38,6 +38,7 @@ from feast import feature_server, flags, flags_helper, utils from feast.base_feature_view import BaseFeatureView from feast.diff.FcoDiff import RegistryDiff +from feast.diff.infra_diff import InfraDiff, diff_infra_protos from feast.entity import Entity from feast.errors import ( EntityNotFoundException, @@ -63,6 +64,7 @@ from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.on_demand_feature_view import OnDemandFeatureView from feast.online_response import OnlineResponse +from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.serving.ServingService_pb2 import ( FieldStatus, @@ -405,7 +407,9 @@ def _get_features( return _feature_refs @log_exceptions_and_usage - def plan(self, desired_repo_objects: RepoContents) -> RegistryDiff: + def plan( + self, desired_repo_objects: RepoContents + ) -> Tuple[RegistryDiff, InfraDiff]: """Dry-run registering objects to metadata store. The plan method dry-runs registering one or more definitions (e.g., Entity, FeatureView), and produces @@ -440,7 +444,7 @@ def plan(self, desired_repo_objects: RepoContents) -> RegistryDiff: ... ttl=timedelta(seconds=86400 * 1), ... batch_source=driver_hourly_stats, ... ) - >>> diff = fs.plan(RepoContents({driver_hourly_stats_view}, set(), set(), {driver}, set())) # register entity and feature view + >>> registry_diff, infra_diff = fs.plan(RepoContents({driver_hourly_stats_view}, set(), set(), {driver}, set())) # register entity and feature view """ current_registry_proto = ( @@ -450,8 +454,21 @@ def plan(self, desired_repo_objects: RepoContents) -> RegistryDiff: ) desired_registry_proto = desired_repo_objects.to_registry_proto() - diffs = Registry.diff_between(current_registry_proto, desired_registry_proto) - return diffs + registry_diff = Registry.diff_between( + current_registry_proto, desired_registry_proto + ) + + current_infra_proto = ( + self._registry.cached_registry_proto.infra.__deepcopy__() + if self._registry.cached_registry_proto + else InfraProto() + ) + new_infra_proto = self._provider.plan_infra( + self.config, desired_registry_proto + ).to_proto() + infra_diff = diff_infra_protos(current_infra_proto, new_infra_proto) + + return (registry_diff, infra_diff) @log_exceptions_and_usage def apply( diff --git a/sdk/python/feast/infra/infra_object.py b/sdk/python/feast/infra/infra_object.py index 282b4bcfab..f21016dea5 100644 --- a/sdk/python/feast/infra/infra_object.py +++ b/sdk/python/feast/infra/infra_object.py @@ -15,13 +15,21 @@ from dataclasses import dataclass, field from typing import Any, List +from feast.errors import FeastInvalidInfraObjectType from feast.importer import import_class +from feast.protos.feast.core.DatastoreTable_pb2 import ( + DatastoreTable as DatastoreTableProto, +) +from feast.protos.feast.core.DynamoDBTable_pb2 import ( + DynamoDBTable as DynamoDBTableProto, +) from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto +from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto DATASTORE_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_stores.datastore.DatastoreTable" DYNAMODB_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_stores.dynamodb.DynamoDBTable" -SQLITE_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_store.sqlite.SqliteTable" +SQLITE_INFRA_OBJECT_CLASS_TYPE = "feast.infra.online_stores.sqlite.SqliteTable" class InfraObject(ABC): @@ -49,7 +57,7 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: infra_object_proto: A protobuf representation of an InfraObject. Raises: - ValueError: The type of InfraObject could not be identified. + FeastInvalidInfraObjectType: The type of InfraObject could not be identified. """ if infra_object_proto.infra_object_class_type: cls = _get_infra_object_class_from_type( @@ -57,7 +65,30 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: ) return cls.from_infra_object_proto(infra_object_proto) - raise ValueError("Could not identify the type of the InfraObject.") + raise FeastInvalidInfraObjectType() + + @staticmethod + def from_proto(infra_object_proto: Any) -> Any: + """ + Converts a protobuf representation of a subclass to an object of that subclass. + + Args: + infra_object_proto: A protobuf representation of an InfraObject. + + Raises: + FeastInvalidInfraObjectType: The type of InfraObject could not be identified. + """ + if isinstance(infra_object_proto, DatastoreTableProto): + infra_object_class_type = DATASTORE_INFRA_OBJECT_CLASS_TYPE + elif isinstance(infra_object_proto, DynamoDBTableProto): + infra_object_class_type = DYNAMODB_INFRA_OBJECT_CLASS_TYPE + elif isinstance(infra_object_proto, SqliteTableProto): + infra_object_class_type = SQLITE_INFRA_OBJECT_CLASS_TYPE + else: + raise FeastInvalidInfraObjectType() + + cls = _get_infra_object_class_from_type(infra_object_class_type) + return cls.from_proto(infra_object_proto) @abstractmethod def update(self): @@ -94,7 +125,7 @@ def to_proto(self) -> InfraProto: """ infra_proto = InfraProto() for infra_object in self.infra_objects: - infra_object_proto = infra_object.to_proto() + infra_object_proto = infra_object.to_infra_object_proto() infra_proto.infra_objects.append(infra_object_proto) return infra_proto diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py index 31c46cf282..060ac64d53 100644 --- a/sdk/python/feast/infra/local.py +++ b/sdk/python/feast/infra/local.py @@ -1,12 +1,13 @@ import uuid from datetime import datetime from pathlib import Path +from typing import List -from feast.feature_view import FeatureView +from feast.infra.infra_object import Infra, InfraObject from feast.infra.passthrough_provider import PassthroughProvider from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.registry_store import RegistryStore -from feast.repo_config import RegistryConfig +from feast.repo_config import RegistryConfig, RepoConfig from feast.usage import log_exceptions_and_usage @@ -15,11 +16,15 @@ class LocalProvider(PassthroughProvider): This class only exists for backwards compatibility. """ - pass - - -def _table_id(project: str, table: FeatureView) -> str: - return f"{project}_{table.name}" + def plan_infra( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> Infra: + infra_objects: List[InfraObject] = self.online_store.plan( + config, desired_registry_proto + ) + infra = Infra() + infra.infra_objects += infra_objects + return infra class LocalRegistryStore(RegistryStore): diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index 348583a202..5a8d4b7180 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -376,6 +376,7 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: name=infra_object_proto.datastore_table.name, ) + # Distinguish between null and empty string, since project_id and namespace are StringValues. if infra_object_proto.datastore_table.HasField("project_id"): datastore_table.project_id = ( infra_object_proto.datastore_table.project_id.value @@ -387,6 +388,20 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: return datastore_table + @staticmethod + def from_proto(datastore_table_proto: DatastoreTableProto) -> Any: + datastore_table = DatastoreTable( + project=datastore_table_proto.project, name=datastore_table_proto.name, + ) + + # Distinguish between null and empty string, since project_id and namespace are StringValues. + if datastore_table_proto.HasField("project_id"): + datastore_table.project_id = datastore_table_proto.project_id.value + if datastore_table_proto.HasField("namespace"): + datastore_table.namespace = datastore_table_proto.namespace.value + + return datastore_table + def update(self): client = _initialize_client(self.project_id, self.namespace) key = client.key("Project", self.project, "Table", self.name) diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 202cfa54bb..b7f8680e1f 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -254,6 +254,12 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: region=infra_object_proto.dynamodb_table.region, ) + @staticmethod + def from_proto(dynamodb_table_proto: DynamoDBTableProto) -> Any: + return DynamoDBTable( + name=dynamodb_table_proto.name, region=dynamodb_table_proto.region, + ) + def update(self): dynamodb_client = _initialize_dynamodb_client(region=self.region) dynamodb_resource = _initialize_dynamodb_resource(region=self.region) diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index b2aa1e46d0..1f177996de 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -18,6 +18,8 @@ from feast import Entity from feast.feature_view import FeatureView +from feast.infra.infra_object import InfraObject +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RepoConfig @@ -92,6 +94,18 @@ def update( ): ... + def plan( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> List[InfraObject]: + """ + Returns the set of InfraObjects required to support the desired registry. + + Args: + config: The RepoConfig for the current FeatureStore. + desired_registry_proto: The desired registry, in proto form. + """ + return [] + @abstractmethod def teardown( self, diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 2dcbf319c3..1e7ecf1024 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -27,6 +27,7 @@ from feast.infra.key_encoding_utils import serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -199,6 +200,21 @@ def update( for table in tables_to_delete: conn.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") + @log_exceptions_and_usage(online_store="sqlite") + def plan( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> List[InfraObject]: + project = config.project + + infra_objects: List[InfraObject] = [ + SqliteTable( + path=self._get_db_path(config), + name=_table_id(project, FeatureView.from_proto(view)), + ) + for view in desired_registry_proto.feature_views + ] + return infra_objects + def teardown( self, config: RepoConfig, @@ -261,6 +277,10 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: name=infra_object_proto.sqlite_table.name, ) + @staticmethod + def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: + return SqliteTable(path=sqlite_table_proto.path, name=sqlite_table_proto.name,) + def update(self): self.conn.execute( f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 3c761f1195..8f9dda9351 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -12,8 +12,10 @@ from feast.entity import Entity from feast.feature_view import DUMMY_ENTITY_ID, FeatureView from feast.importer import import_class +from feast.infra.infra_object import Infra from feast.infra.offline_stores.offline_store import RetrievalJob from feast.on_demand_feature_view import OnDemandFeatureView +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.registry import Registry @@ -61,6 +63,18 @@ def update_infra( """ ... + def plan_infra( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> Infra: + """ + Returns the Infra required to support the desired registry. + + Args: + config: The RepoConfig for the current FeatureStore. + desired_registry_proto: The desired registry, in proto form. + """ + return Infra() + @abc.abstractmethod def teardown_infra( self, project: str, tables: Sequence[FeatureView], entities: Sequence[Entity], diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 9299a36123..3e9ddb6e30 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -127,20 +127,20 @@ def plan(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool) for data_source in data_sources: data_source.validate(store.config) - diff = store.plan(repo) + registry_diff, _ = store.plan(repo) views_to_delete = [ v - for v in diff.fco_diffs + for v in registry_diff.fco_diffs if v.fco_type == "feature view" and v.transition_type == TransitionType.DELETE ] views_to_keep = [ v - for v in diff.fco_diffs + for v in registry_diff.fco_diffs if v.fco_type == "feature view" and v.transition_type in {TransitionType.CREATE, TransitionType.UNCHANGED} ] - log_cli_output(diff, views_to_delete, views_to_keep) + log_cli_output(registry_diff, views_to_delete, views_to_keep) def _prepare_registry_and_repo(repo_config, repo_path): From 1b98ec94e3573991627d561d6d207126a40a21cf Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Fri, 14 Jan 2022 13:52:13 -0800 Subject: [PATCH 12/15] =?UTF-8?q?replace=20GetOnlineFeaturesResponse=20wit?= =?UTF-8?q?h=20GetOnlineFeaturesResponseV2=20in=E2=80=A6=20(#2214)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * replace GetOnlineFeaturesResponse with GetOnlineFeaturesResponseV2 in Python Signed-off-by: Tsotne Tabidze * Fix unit tests Signed-off-by: Tsotne Tabidze * Fix java compilation & python integration tests Signed-off-by: Tsotne Tabidze * Fix integration tests Signed-off-by: Tsotne Tabidze --- go.mod | 4 +- go.sum | 7 + .../logging/entry/AuditLogEntryTest.java | 26 +- .../src/main/java/dev/feast/FeastClient.java | 4 +- .../test/java/dev/feast/FeastClientTest.java | 14 +- .../ServingServiceGRpcController.java | 4 +- .../ServingServiceRestController.java | 2 +- .../grpc/OnlineServingGrpcServiceV2.java | 2 +- .../service/OnlineServingServiceV2.java | 12 +- .../service/OnlineTransformationService.java | 4 +- .../serving/service/ServingServiceV2.java | 6 +- .../service/TransformationService.java | 4 +- .../util/mappers/ResponseJSONMapper.java | 4 +- .../feast/serving/it/ServingBaseTests.java | 6 +- .../service/OnlineServingServiceTest.java | 32 +- protos/feast/serving/ServingService.proto | 15 +- sdk/go/client_test.go | 4 +- sdk/go/mocks/serving_mock.go | 4 +- .../protos/feast/serving/ServingService.pb.go | 415 ++++++------------ sdk/go/response.go | 2 +- sdk/go/response_test.go | 4 +- sdk/python/feast/feature_store.py | 152 ++++--- .../feast/infra/online_stores/dynamodb.py | 2 +- sdk/python/feast/online_response.py | 58 +-- .../online_store/test_e2e_local.py | 9 +- .../online_store/test_universal_online.py | 6 +- sdk/python/tests/unit/test_proto_json.py | 81 ++-- 27 files changed, 361 insertions(+), 522 deletions(-) diff --git a/go.mod b/go.mod index f4a1455056..109666b762 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,8 @@ require ( go.opencensus.io v0.22.3 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d - golang.org/x/tools v0.1.7 // indirect + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f + golang.org/x/tools v0.1.8 // indirect google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.27.1 // indirect diff --git a/go.sum b/go.sum index 5e87ccf6db..8b0c2677f3 100644 --- a/go.sum +++ b/go.sum @@ -345,6 +345,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -386,6 +387,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= @@ -415,6 +417,7 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -454,6 +457,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -464,6 +468,7 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -525,6 +530,8 @@ golang.org/x/tools v0.0.0-20201124005743-911501bfb504 h1:jOKV2ysikH1GANB7t2Lotmh golang.org/x/tools v0.0.0-20201124005743-911501bfb504/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= +golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java b/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java index 0c96ee9c56..bc3dcbcf74 100644 --- a/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java +++ b/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java @@ -21,11 +21,12 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.protobuf.Timestamp; import feast.common.logging.entry.LogResource.ResourceType; +import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto.Value; import io.grpc.Status; import java.util.Arrays; @@ -50,15 +51,24 @@ public List getTestAuditLogs() { GetOnlineFeaturesResponse responseSpec = GetOnlineFeaturesResponse.newBuilder() - .addAllFieldValues( + .setMetadata( + ServingAPIProto.GetOnlineFeaturesResponseMetadata.newBuilder() + .setFeatureNames( + ServingAPIProto.FeatureList.newBuilder() + .addAllVal( + Arrays.asList( + "featuretable_1:feature_1", "featuretable_1:feature2")))) + .addAllResults( Arrays.asList( - FieldValues.newBuilder() - .putFields( - "featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build()) + GetOnlineFeaturesResponse.FeatureVector.newBuilder() + .addValues(Value.newBuilder().setInt32Val(32).build()) + .addStatuses(ServingAPIProto.FieldStatus.PRESENT) + .addEventTimestamps(Timestamp.newBuilder().build()) .build(), - FieldValues.newBuilder() - .putFields( - "featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build()) + GetOnlineFeaturesResponse.FeatureVector.newBuilder() + .addValues(Value.newBuilder().setInt32Val(64).build()) + .addStatuses(ServingAPIProto.FieldStatus.PRESENT) + .addEventTimestamps(Timestamp.newBuilder().build()) .build())) .build(); diff --git a/java/sdk/java/src/main/java/dev/feast/FeastClient.java b/java/sdk/java/src/main/java/dev/feast/FeastClient.java index e9aaab151a..c10a76ecf8 100644 --- a/java/sdk/java/src/main/java/dev/feast/FeastClient.java +++ b/java/sdk/java/src/main/java/dev/feast/FeastClient.java @@ -21,7 +21,7 @@ import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponseV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; import feast.proto.types.ValueProto; @@ -129,7 +129,7 @@ public List getOnlineFeatures(List featureRefs, List entities) requestBuilder.putAllEntities(getEntityValuesMap(entities)); - GetOnlineFeaturesResponseV2 response = stub.getOnlineFeatures(requestBuilder.build()); + GetOnlineFeaturesResponse response = stub.getOnlineFeatures(requestBuilder.build()); List results = Lists.newArrayList(); if (response.getResultsCount() == 0) { diff --git a/java/sdk/java/src/test/java/dev/feast/FeastClientTest.java b/java/sdk/java/src/test/java/dev/feast/FeastClientTest.java index 3de5142a85..1dfb9989c9 100644 --- a/java/sdk/java/src/test/java/dev/feast/FeastClientTest.java +++ b/java/sdk/java/src/test/java/dev/feast/FeastClientTest.java @@ -24,7 +24,7 @@ import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponseV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; @@ -57,7 +57,7 @@ public class FeastClientTest { @Override public void getOnlineFeatures( GetOnlineFeaturesRequest request, - StreamObserver responseObserver) { + StreamObserver responseObserver) { if (!request.equals(FeastClientTest.getFakeRequest())) { responseObserver.onError(Status.FAILED_PRECONDITION.asRuntimeException()); } @@ -137,22 +137,22 @@ private static GetOnlineFeaturesRequest getFakeRequest() { .build(); } - private static GetOnlineFeaturesResponseV2 getFakeResponse() { - return GetOnlineFeaturesResponseV2.newBuilder() + private static GetOnlineFeaturesResponse getFakeResponse() { + return GetOnlineFeaturesResponse.newBuilder() .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(strValue("david")) .addStatuses(FieldStatus.PRESENT) .addEventTimestamps(Timestamp.newBuilder()) .build()) .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(intValue(3)) .addStatuses(FieldStatus.PRESENT) .addEventTimestamps(Timestamp.newBuilder()) .build()) .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(Value.newBuilder().build()) .addStatuses(FieldStatus.NULL_VALUE) .addEventTimestamps(Timestamp.newBuilder()) diff --git a/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 0f4ef7b5ae..bc6af8ecce 100644 --- a/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -60,12 +60,12 @@ public void getFeastServingInfo( @Override public void getOnlineFeatures( ServingAPIProto.GetOnlineFeaturesRequest request, - StreamObserver responseObserver) { + StreamObserver responseObserver) { try { // authorize for the project in request object. RequestHelper.validateOnlineRequest(request); Span span = tracer.buildSpan("getOnlineFeaturesV2").start(); - ServingAPIProto.GetOnlineFeaturesResponseV2 onlineFeatures = + ServingAPIProto.GetOnlineFeaturesResponse onlineFeatures = servingServiceV2.getOnlineFeatures(request); if (span != null) { span.finish(); diff --git a/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java b/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java index fe8f13d8bc..1983f3ebce 100644 --- a/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java +++ b/java/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java @@ -54,7 +54,7 @@ public GetFeastServingInfoResponse getInfo() { public List> getOnlineFeatures( @RequestBody ServingAPIProto.GetOnlineFeaturesRequest request) { RequestHelper.validateOnlineRequest(request); - ServingAPIProto.GetOnlineFeaturesResponseV2 onlineFeatures = + ServingAPIProto.GetOnlineFeaturesResponse onlineFeatures = servingService.getOnlineFeatures(request); return mapGetOnlineFeaturesResponse(onlineFeatures); } diff --git a/java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java b/java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java index f3a35d1d0f..8d82a1f182 100644 --- a/java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java +++ b/java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java @@ -41,7 +41,7 @@ public void getFeastServingInfo( @Override public void getOnlineFeatures( ServingAPIProto.GetOnlineFeaturesRequest request, - StreamObserver responseObserver) { + StreamObserver responseObserver) { responseObserver.onNext(this.servingServiceV2.getOnlineFeatures(request)); responseObserver.onCompleted(); } diff --git a/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 5774dc361a..3e62d4db75 100644 --- a/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -72,7 +72,7 @@ public GetFeastServingInfoResponse getFeastServingInfo( } @Override - public ServingAPIProto.GetOnlineFeaturesResponseV2 getOnlineFeatures( + public ServingAPIProto.GetOnlineFeaturesResponse getOnlineFeatures( ServingAPIProto.GetOnlineFeaturesRequest request) { // Split all feature references into non-ODFV (e.g. batch and stream) references and ODFV. List allFeatureReferences = getFeaturesList(request); @@ -132,8 +132,8 @@ public ServingAPIProto.GetOnlineFeaturesResponseV2 getOnlineFeatures( Span postProcessingSpan = tracer.buildSpan("postProcessing").start(); - ServingAPIProto.GetOnlineFeaturesResponseV2.Builder responseBuilder = - ServingAPIProto.GetOnlineFeaturesResponseV2.newBuilder(); + ServingAPIProto.GetOnlineFeaturesResponse.Builder responseBuilder = + ServingAPIProto.GetOnlineFeaturesResponse.newBuilder(); Timestamp now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); Timestamp nullTimestamp = Timestamp.newBuilder().build(); @@ -147,7 +147,7 @@ public ServingAPIProto.GetOnlineFeaturesResponseV2 getOnlineFeatures( Duration maxAge = this.registryRepository.getMaxAge(featureReference); - ServingAPIProto.GetOnlineFeaturesResponseV2.FeatureVector.Builder vectorBuilder = + ServingAPIProto.GetOnlineFeaturesResponse.FeatureVector.Builder vectorBuilder = responseBuilder.addResultsBuilder(); for (int rowIdx = 0; rowIdx < features.size(); rowIdx++) { @@ -262,7 +262,7 @@ private void populateOnDemandFeatures( List retrievedFeatureReferences, ServingAPIProto.GetOnlineFeaturesRequest request, List> features, - ServingAPIProto.GetOnlineFeaturesResponseV2.Builder responseBuilder) { + ServingAPIProto.GetOnlineFeaturesResponse.Builder responseBuilder) { List>> onDemandContext = request.getRequestContextMap().entrySet().stream() @@ -383,7 +383,7 @@ private void populateHistogramMetrics( */ private void populateCountMetrics( FeatureReferenceV2 featureRef, - ServingAPIProto.GetOnlineFeaturesResponseV2.FeatureVectorOrBuilder featureVector) { + ServingAPIProto.GetOnlineFeaturesResponse.FeatureVectorOrBuilder featureVector) { String featureRefString = Feature.getFeatureReference(featureRef); featureVector .getStatusesList() diff --git a/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java b/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java index bfe717aa96..a535eacb9e 100644 --- a/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java +++ b/java/serving/src/main/java/feast/serving/service/OnlineTransformationService.java @@ -189,7 +189,7 @@ public void processTransformFeaturesResponse( transformFeaturesResponse, String onDemandFeatureViewName, Set onDemandFeatureStringReferences, - ServingAPIProto.GetOnlineFeaturesResponseV2.Builder responseBuilder) { + ServingAPIProto.GetOnlineFeaturesResponse.Builder responseBuilder) { try { BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); ArrowFileReader reader = @@ -219,7 +219,7 @@ public void processTransformFeaturesResponse( FieldVector fieldVector = readBatch.getVector(field); int valueCount = fieldVector.getValueCount(); - ServingAPIProto.GetOnlineFeaturesResponseV2.FeatureVector.Builder vectorBuilder = + ServingAPIProto.GetOnlineFeaturesResponse.FeatureVector.Builder vectorBuilder = responseBuilder.addResultsBuilder(); List valueList = Lists.newArrayListWithExpectedSize(valueCount); diff --git a/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java b/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java index 096b155a0e..4a44f4e09e 100644 --- a/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java +++ b/java/serving/src/main/java/feast/serving/service/ServingServiceV2.java @@ -40,9 +40,9 @@ ServingAPIProto.GetFeastServingInfoResponse getFeastServingInfo( * *

This request is fulfilled synchronously. * - * @return {@link feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponseV2} with list of - * {@link feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponseV2.FeatureVector}. + * @return {@link feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse} with list of + * {@link feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FeatureVector}. */ - ServingAPIProto.GetOnlineFeaturesResponseV2 getOnlineFeatures( + ServingAPIProto.GetOnlineFeaturesResponse getOnlineFeatures( ServingAPIProto.GetOnlineFeaturesRequest getFeaturesRequest); } diff --git a/java/serving/src/main/java/feast/serving/service/TransformationService.java b/java/serving/src/main/java/feast/serving/service/TransformationService.java index 36cce43e0d..e47e847f9e 100644 --- a/java/serving/src/main/java/feast/serving/service/TransformationService.java +++ b/java/serving/src/main/java/feast/serving/service/TransformationService.java @@ -66,13 +66,13 @@ public interface TransformationService { * @param transformFeaturesResponse response to be processed * @param onDemandFeatureViewName name of ODFV to which the response corresponds * @param onDemandFeatureStringReferences set of all ODFV references that should be kept - * @param responseBuilder {@link ServingAPIProto.GetOnlineFeaturesResponseV2.Builder} + * @param responseBuilder {@link ServingAPIProto.GetOnlineFeaturesResponse.Builder} */ void processTransformFeaturesResponse( TransformFeaturesResponse transformFeaturesResponse, String onDemandFeatureViewName, Set onDemandFeatureStringReferences, - ServingAPIProto.GetOnlineFeaturesResponseV2.Builder responseBuilder); + ServingAPIProto.GetOnlineFeaturesResponse.Builder responseBuilder); /** * Serialize data into Arrow IPC format, to be sent to the Python feature transformation server. diff --git a/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java index 1e82bf864c..3ab9f43c34 100644 --- a/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java +++ b/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java @@ -26,14 +26,14 @@ public class ResponseJSONMapper { public static List> mapGetOnlineFeaturesResponse( - ServingAPIProto.GetOnlineFeaturesResponseV2 response) { + ServingAPIProto.GetOnlineFeaturesResponse response) { return response.getResultsList().stream() .map(fieldValues -> convertFieldValuesToMap(fieldValues)) .collect(Collectors.toList()); } private static Map convertFieldValuesToMap( - ServingAPIProto.GetOnlineFeaturesResponseV2.FeatureVector vec) { + ServingAPIProto.GetOnlineFeaturesResponse.FeatureVector vec) { return Map.of( "values", vec.getValuesList().stream() diff --git a/java/serving/src/test/java/feast/serving/it/ServingBaseTests.java b/java/serving/src/test/java/feast/serving/it/ServingBaseTests.java index 4d4272324e..cd732ce1dd 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingBaseTests.java +++ b/java/serving/src/test/java/feast/serving/it/ServingBaseTests.java @@ -74,7 +74,7 @@ private static RegistryProto.Registry readLocalRegistry() { @Test public void shouldGetOnlineFeatures() { - ServingAPIProto.GetOnlineFeaturesResponseV2 featureResponse = + ServingAPIProto.GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(buildOnlineRequest(1005)); assertEquals(2, featureResponse.getResultsCount()); @@ -96,7 +96,7 @@ public void shouldGetOnlineFeatures() { @Test public void shouldGetOnlineFeaturesWithOutsideMaxAgeStatus() { - ServingAPIProto.GetOnlineFeaturesResponseV2 featureResponse = + ServingAPIProto.GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(buildOnlineRequest(1001)); assertEquals(2, featureResponse.getResultsCount()); @@ -113,7 +113,7 @@ public void shouldGetOnlineFeaturesWithOutsideMaxAgeStatus() { @Test public void shouldGetOnlineFeaturesWithNotFoundStatus() { - ServingAPIProto.GetOnlineFeaturesResponseV2 featureResponse = + ServingAPIProto.GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(buildOnlineRequest(-1)); assertEquals(2, featureResponse.getResultsCount()); diff --git a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 4234e9dce3..64d2e20c9b 100644 --- a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -30,7 +30,7 @@ import feast.proto.core.FeatureViewProto; import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.FieldStatus; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponseV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.types.ValueProto; import feast.serving.registry.Registry; import feast.serving.registry.RegistryRepository; @@ -173,10 +173,10 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - GetOnlineFeaturesResponseV2 expected = - GetOnlineFeaturesResponseV2.newBuilder() + GetOnlineFeaturesResponse expected = + GetOnlineFeaturesResponse.newBuilder() .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(createStrValue("1")) .addValues(createStrValue("3")) .addStatuses(FieldStatus.PRESENT) @@ -184,7 +184,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { .addEventTimestamps(now) .addEventTimestamps(now)) .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(createStrValue("2")) .addValues(createStrValue("4")) .addStatuses(FieldStatus.PRESENT) @@ -198,7 +198,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { .addVal("featureview_1:feature_1") .addVal("featureview_1:feature_2"))) .build(); - ServingAPIProto.GetOnlineFeaturesResponseV2 actual = + ServingAPIProto.GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } @@ -240,10 +240,10 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - GetOnlineFeaturesResponseV2 expected = - GetOnlineFeaturesResponseV2.newBuilder() + GetOnlineFeaturesResponse expected = + GetOnlineFeaturesResponse.newBuilder() .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(createStrValue("1")) .addValues(createEmptyValue()) .addStatuses(FieldStatus.PRESENT) @@ -251,7 +251,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { .addEventTimestamps(now) .addEventTimestamps(Timestamp.newBuilder().build())) .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(createStrValue("2")) .addValues(createStrValue("5")) .addStatuses(FieldStatus.PRESENT) @@ -265,7 +265,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { .addVal("featureview_1:feature_1") .addVal("featureview_1:feature_2"))) .build(); - GetOnlineFeaturesResponseV2 actual = onlineServingServiceV2.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } @@ -317,10 +317,10 @@ public void shouldReturnResponseWithValuesAndMetadataIfMaxAgeIsExceeded() { when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - GetOnlineFeaturesResponseV2 expected = - GetOnlineFeaturesResponseV2.newBuilder() + GetOnlineFeaturesResponse expected = + GetOnlineFeaturesResponse.newBuilder() .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(createStrValue("6")) .addValues(createStrValue("6")) .addStatuses(FieldStatus.OUTSIDE_MAX_AGE) @@ -328,7 +328,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfMaxAgeIsExceeded() { .addEventTimestamps(Timestamp.newBuilder().setSeconds(1).build()) .addEventTimestamps(Timestamp.newBuilder().setSeconds(1).build())) .addResults( - GetOnlineFeaturesResponseV2.FeatureVector.newBuilder() + GetOnlineFeaturesResponse.FeatureVector.newBuilder() .addValues(createStrValue("2")) .addValues(createStrValue("2")) .addStatuses(FieldStatus.PRESENT) @@ -342,7 +342,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfMaxAgeIsExceeded() { .addVal("featureview_1:feature_1") .addVal("featureview_1:feature_2"))) .build(); - GetOnlineFeaturesResponseV2 actual = onlineServingServiceV2.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 7d45e61a5e..6c551a97ba 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -30,7 +30,7 @@ service ServingService { rpc GetFeastServingInfo (GetFeastServingInfoRequest) returns (GetFeastServingInfoResponse); // Get online features synchronously. - rpc GetOnlineFeatures (GetOnlineFeaturesRequest) returns (GetOnlineFeaturesResponseV2); + rpc GetOnlineFeatures (GetOnlineFeaturesRequest) returns (GetOnlineFeaturesResponse); } message GetFeastServingInfoRequest {} @@ -95,19 +95,6 @@ message GetOnlineFeaturesRequest { } message GetOnlineFeaturesResponse { - // Feature values retrieved from feast. - repeated FieldValues field_values = 1; - - message FieldValues { - // Map of feature or entity name to feature/entity values. - // Timestamps are not returned in this response. - map fields = 1; - // Map of feature or entity name to feature/entity statuses/metadata. - map statuses = 2; - } -} - -message GetOnlineFeaturesResponseV2 { GetOnlineFeaturesResponseMetadata metadata = 1; // Length of "results" array should match length of requested features. diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 95be34af73..cb15f66654 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -33,8 +33,8 @@ func TestGetOnlineFeatures(t *testing.T) { Project: "driver_project", }, want: OnlineFeaturesResponse{ - RawResponse: &serving.GetOnlineFeaturesResponseV2{ - Results: []*serving.GetOnlineFeaturesResponseV2_FeatureVector{ + RawResponse: &serving.GetOnlineFeaturesResponse{ + Results: []*serving.GetOnlineFeaturesResponse_FeatureVector{ { Values: []*types.Value{Int64Val(1)}, Statuses: []serving.FieldStatus{ diff --git a/sdk/go/mocks/serving_mock.go b/sdk/go/mocks/serving_mock.go index 57ee0c1ea4..038d49f5e5 100644 --- a/sdk/go/mocks/serving_mock.go +++ b/sdk/go/mocks/serving_mock.go @@ -57,14 +57,14 @@ func (mr *MockServingServiceClientMockRecorder) GetFeastServingInfo(arg0, arg1 i } // GetOnlineFeaturesV2 mocks base method -func (m *MockServingServiceClient) GetOnlineFeatures(arg0 context.Context, arg1 *serving.GetOnlineFeaturesRequest, arg2 ...grpc.CallOption) (*serving.GetOnlineFeaturesResponseV2, error) { +func (m *MockServingServiceClient) GetOnlineFeatures(arg0 context.Context, arg1 *serving.GetOnlineFeaturesRequest, arg2 ...grpc.CallOption) (*serving.GetOnlineFeaturesResponse, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} for _, a := range arg2 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetOnlineFeatures", varargs...) - ret0, _ := ret[0].(*serving.GetOnlineFeaturesResponseV2) + ret0, _ := ret[0].(*serving.GetOnlineFeaturesResponse) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index 68e771a31b..b367a307f4 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -380,7 +380,10 @@ type GetOnlineFeaturesRequest struct { // A map of entity name -> list of values Entities map[string]*types.RepeatedValue `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` FullFeatureNames bool `protobuf:"varint,4,opt,name=full_feature_names,json=fullFeatureNames,proto3" json:"full_feature_names,omitempty"` - RequestContext map[string]*types.RepeatedValue `protobuf:"bytes,5,rep,name=request_context,json=requestContext,proto3" json:"request_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Context for OnDemand Feature Transformation + // (was moved to dedicated parameter to avoid unnecessary separation logic on serving side) + // A map of variable name -> list of values + RequestContext map[string]*types.RepeatedValue `protobuf:"bytes,5,rep,name=request_context,json=requestContext,proto3" json:"request_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *GetOnlineFeaturesRequest) Reset() { @@ -478,8 +481,10 @@ type GetOnlineFeaturesResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Feature values retrieved from feast. - FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` + Metadata *GetOnlineFeaturesResponseMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // Length of "results" array should match length of requested features. + // We also preserve the same order of features here as in metadata.feature_names + Results []*GetOnlineFeaturesResponse_FeatureVector `protobuf:"bytes,2,rep,name=results,proto3" json:"results,omitempty"` } func (x *GetOnlineFeaturesResponse) Reset() { @@ -514,62 +519,14 @@ func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6} } -func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { - if x != nil { - return x.FieldValues - } - return nil -} - -type GetOnlineFeaturesResponseV2 struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Metadata *GetOnlineFeaturesResponseMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` - Results []*GetOnlineFeaturesResponseV2_FeatureVector `protobuf:"bytes,2,rep,name=results,proto3" json:"results,omitempty"` -} - -func (x *GetOnlineFeaturesResponseV2) Reset() { - *x = GetOnlineFeaturesResponseV2{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOnlineFeaturesResponseV2) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOnlineFeaturesResponseV2) ProtoMessage() {} - -func (x *GetOnlineFeaturesResponseV2) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOnlineFeaturesResponseV2.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponseV2) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{7} -} - -func (x *GetOnlineFeaturesResponseV2) GetMetadata() *GetOnlineFeaturesResponseMetadata { +func (x *GetOnlineFeaturesResponse) GetMetadata() *GetOnlineFeaturesResponseMetadata { if x != nil { return x.Metadata } return nil } -func (x *GetOnlineFeaturesResponseV2) GetResults() []*GetOnlineFeaturesResponseV2_FeatureVector { +func (x *GetOnlineFeaturesResponse) GetResults() []*GetOnlineFeaturesResponse_FeatureVector { if x != nil { return x.Results } @@ -587,7 +544,7 @@ type GetOnlineFeaturesResponseMetadata struct { func (x *GetOnlineFeaturesResponseMetadata) Reset() { *x = GetOnlineFeaturesResponseMetadata{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[8] + mi := &file_feast_serving_ServingService_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -600,7 +557,7 @@ func (x *GetOnlineFeaturesResponseMetadata) String() string { func (*GetOnlineFeaturesResponseMetadata) ProtoMessage() {} func (x *GetOnlineFeaturesResponseMetadata) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[8] + mi := &file_feast_serving_ServingService_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -613,7 +570,7 @@ func (x *GetOnlineFeaturesResponseMetadata) ProtoReflect() protoreflect.Message // Deprecated: Use GetOnlineFeaturesResponseMetadata.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesResponseMetadata) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{8} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{7} } func (x *GetOnlineFeaturesResponseMetadata) GetFeatureNames() *FeatureList { @@ -638,7 +595,7 @@ type GetOnlineFeaturesRequestV2_EntityRow struct { func (x *GetOnlineFeaturesRequestV2_EntityRow) Reset() { *x = GetOnlineFeaturesRequestV2_EntityRow{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[9] + mi := &file_feast_serving_ServingService_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -651,7 +608,7 @@ func (x *GetOnlineFeaturesRequestV2_EntityRow) String() string { func (*GetOnlineFeaturesRequestV2_EntityRow) ProtoMessage() {} func (x *GetOnlineFeaturesRequestV2_EntityRow) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[9] + mi := &file_feast_serving_ServingService_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -681,65 +638,7 @@ func (x *GetOnlineFeaturesRequestV2_EntityRow) GetFields() map[string]*types.Val return nil } -type GetOnlineFeaturesResponse_FieldValues struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Map of feature or entity name to feature/entity values. - // Timestamps are not returned in this response. - Fields map[string]*types.Value `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - // Map of feature or entity name to feature/entity statuses/metadata. - Statuses map[string]FieldStatus `protobuf:"bytes,2,rep,name=statuses,proto3" json:"statuses,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=feast.serving.FieldStatus"` -} - -func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { - *x = GetOnlineFeaturesResponse_FieldValues{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOnlineFeaturesResponse_FieldValues) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOnlineFeaturesResponse_FieldValues) ProtoMessage() {} - -func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOnlineFeaturesResponse_FieldValues.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse_FieldValues) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6, 0} -} - -func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Value { - if x != nil { - return x.Fields - } - return nil -} - -func (x *GetOnlineFeaturesResponse_FieldValues) GetStatuses() map[string]FieldStatus { - if x != nil { - return x.Statuses - } - return nil -} - -type GetOnlineFeaturesResponseV2_FeatureVector struct { +type GetOnlineFeaturesResponse_FeatureVector struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -749,23 +648,23 @@ type GetOnlineFeaturesResponseV2_FeatureVector struct { EventTimestamps []*timestamppb.Timestamp `protobuf:"bytes,3,rep,name=event_timestamps,json=eventTimestamps,proto3" json:"event_timestamps,omitempty"` } -func (x *GetOnlineFeaturesResponseV2_FeatureVector) Reset() { - *x = GetOnlineFeaturesResponseV2_FeatureVector{} +func (x *GetOnlineFeaturesResponse_FeatureVector) Reset() { + *x = GetOnlineFeaturesResponse_FeatureVector{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[16] + mi := &file_feast_serving_ServingService_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetOnlineFeaturesResponseV2_FeatureVector) String() string { +func (x *GetOnlineFeaturesResponse_FeatureVector) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetOnlineFeaturesResponseV2_FeatureVector) ProtoMessage() {} +func (*GetOnlineFeaturesResponse_FeatureVector) ProtoMessage() {} -func (x *GetOnlineFeaturesResponseV2_FeatureVector) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[16] +func (x *GetOnlineFeaturesResponse_FeatureVector) ProtoReflect() protoreflect.Message { + mi := &file_feast_serving_ServingService_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -776,26 +675,26 @@ func (x *GetOnlineFeaturesResponseV2_FeatureVector) ProtoReflect() protoreflect. return mi.MessageOf(x) } -// Deprecated: Use GetOnlineFeaturesResponseV2_FeatureVector.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponseV2_FeatureVector) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{7, 0} +// Deprecated: Use GetOnlineFeaturesResponse_FeatureVector.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse_FeatureVector) Descriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6, 0} } -func (x *GetOnlineFeaturesResponseV2_FeatureVector) GetValues() []*types.Value { +func (x *GetOnlineFeaturesResponse_FeatureVector) GetValues() []*types.Value { if x != nil { return x.Values } return nil } -func (x *GetOnlineFeaturesResponseV2_FeatureVector) GetStatuses() []FieldStatus { +func (x *GetOnlineFeaturesResponse_FeatureVector) GetStatuses() []FieldStatus { if x != nil { return x.Statuses } return nil } -func (x *GetOnlineFeaturesResponseV2_FeatureVector) GetEventTimestamps() []*timestamppb.Timestamp { +func (x *GetOnlineFeaturesResponse_FeatureVector) GetEventTimestamps() []*timestamppb.Timestamp { if x != nil { return x.EventTimestamps } @@ -888,94 +787,63 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, - 0x64, 0x22, 0xe6, 0x03, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, + 0x64, 0x22, 0xf8, 0x02, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x57, 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, + 0x4c, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x30, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x50, 0x0a, + 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, + 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, + 0xba, 0x01, 0x0a, 0x0d, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x56, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x12, 0x2a, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x36, 0x0a, + 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0e, 0x32, + 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x73, 0x22, 0x64, 0x0a, 0x21, + 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x12, 0x3f, 0x0a, 0x0d, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x73, 0x2a, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, + 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, + 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, + 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, + 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x32, + 0xe6, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, + 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x66, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0xef, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, - 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x65, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x1a, 0x57, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfc, 0x02, 0x0a, 0x1b, 0x47, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x56, 0x32, 0x12, 0x4c, 0x0a, 0x08, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, - 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x56, 0x32, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x56, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0xba, 0x01, 0x0a, - 0x0d, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x2a, - 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x36, 0x0a, 0x08, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x65, 0x73, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x73, 0x22, 0x64, 0x0a, 0x21, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3f, - 0x0a, 0x0d, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x2a, - 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, - 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, - 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, - 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, - 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, 0x54, 0x53, 0x49, - 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x32, 0xe8, 0x01, 0x0a, - 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, - 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, - 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x56, 0x32, 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, - 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, - 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, + 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, + 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -991,7 +859,7 @@ func file_feast_serving_ServingService_proto_rawDescGZIP() []byte { } var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_feast_serving_ServingService_proto_goTypes = []interface{}{ (FieldStatus)(0), // 0: feast.serving.FieldStatus (*GetFeastServingInfoRequest)(nil), // 1: feast.serving.GetFeastServingInfoRequest @@ -1001,51 +869,42 @@ var file_feast_serving_ServingService_proto_goTypes = []interface{}{ (*FeatureList)(nil), // 5: feast.serving.FeatureList (*GetOnlineFeaturesRequest)(nil), // 6: feast.serving.GetOnlineFeaturesRequest (*GetOnlineFeaturesResponse)(nil), // 7: feast.serving.GetOnlineFeaturesResponse - (*GetOnlineFeaturesResponseV2)(nil), // 8: feast.serving.GetOnlineFeaturesResponseV2 - (*GetOnlineFeaturesResponseMetadata)(nil), // 9: feast.serving.GetOnlineFeaturesResponseMetadata - (*GetOnlineFeaturesRequestV2_EntityRow)(nil), // 10: feast.serving.GetOnlineFeaturesRequestV2.EntityRow - nil, // 11: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry - nil, // 12: feast.serving.GetOnlineFeaturesRequest.EntitiesEntry - nil, // 13: feast.serving.GetOnlineFeaturesRequest.RequestContextEntry - (*GetOnlineFeaturesResponse_FieldValues)(nil), // 14: feast.serving.GetOnlineFeaturesResponse.FieldValues - nil, // 15: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - nil, // 16: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry - (*GetOnlineFeaturesResponseV2_FeatureVector)(nil), // 17: feast.serving.GetOnlineFeaturesResponseV2.FeatureVector - (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp - (*types.Value)(nil), // 19: feast.types.Value - (*types.RepeatedValue)(nil), // 20: feast.types.RepeatedValue + (*GetOnlineFeaturesResponseMetadata)(nil), // 8: feast.serving.GetOnlineFeaturesResponseMetadata + (*GetOnlineFeaturesRequestV2_EntityRow)(nil), // 9: feast.serving.GetOnlineFeaturesRequestV2.EntityRow + nil, // 10: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry + nil, // 11: feast.serving.GetOnlineFeaturesRequest.EntitiesEntry + nil, // 12: feast.serving.GetOnlineFeaturesRequest.RequestContextEntry + (*GetOnlineFeaturesResponse_FeatureVector)(nil), // 13: feast.serving.GetOnlineFeaturesResponse.FeatureVector + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp + (*types.Value)(nil), // 15: feast.types.Value + (*types.RepeatedValue)(nil), // 16: feast.types.RepeatedValue } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 3, // 0: feast.serving.GetOnlineFeaturesRequestV2.features:type_name -> feast.serving.FeatureReferenceV2 - 10, // 1: feast.serving.GetOnlineFeaturesRequestV2.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow + 9, // 1: feast.serving.GetOnlineFeaturesRequestV2.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow 5, // 2: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureList - 12, // 3: feast.serving.GetOnlineFeaturesRequest.entities:type_name -> feast.serving.GetOnlineFeaturesRequest.EntitiesEntry - 13, // 4: feast.serving.GetOnlineFeaturesRequest.request_context:type_name -> feast.serving.GetOnlineFeaturesRequest.RequestContextEntry - 14, // 5: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues - 9, // 6: feast.serving.GetOnlineFeaturesResponseV2.metadata:type_name -> feast.serving.GetOnlineFeaturesResponseMetadata - 17, // 7: feast.serving.GetOnlineFeaturesResponseV2.results:type_name -> feast.serving.GetOnlineFeaturesResponseV2.FeatureVector - 5, // 8: feast.serving.GetOnlineFeaturesResponseMetadata.feature_names:type_name -> feast.serving.FeatureList - 18, // 9: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.timestamp:type_name -> google.protobuf.Timestamp - 11, // 10: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry - 19, // 11: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 20, // 12: feast.serving.GetOnlineFeaturesRequest.EntitiesEntry.value:type_name -> feast.types.RepeatedValue - 20, // 13: feast.serving.GetOnlineFeaturesRequest.RequestContextEntry.value:type_name -> feast.types.RepeatedValue - 15, // 14: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - 16, // 15: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry - 19, // 16: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value - 0, // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.FieldStatus - 19, // 18: feast.serving.GetOnlineFeaturesResponseV2.FeatureVector.values:type_name -> feast.types.Value - 0, // 19: feast.serving.GetOnlineFeaturesResponseV2.FeatureVector.statuses:type_name -> feast.serving.FieldStatus - 18, // 20: feast.serving.GetOnlineFeaturesResponseV2.FeatureVector.event_timestamps:type_name -> google.protobuf.Timestamp - 1, // 21: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest - 6, // 22: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest - 2, // 23: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse - 8, // 24: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponseV2 - 23, // [23:25] is the sub-list for method output_type - 21, // [21:23] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 11, // 3: feast.serving.GetOnlineFeaturesRequest.entities:type_name -> feast.serving.GetOnlineFeaturesRequest.EntitiesEntry + 12, // 4: feast.serving.GetOnlineFeaturesRequest.request_context:type_name -> feast.serving.GetOnlineFeaturesRequest.RequestContextEntry + 8, // 5: feast.serving.GetOnlineFeaturesResponse.metadata:type_name -> feast.serving.GetOnlineFeaturesResponseMetadata + 13, // 6: feast.serving.GetOnlineFeaturesResponse.results:type_name -> feast.serving.GetOnlineFeaturesResponse.FeatureVector + 5, // 7: feast.serving.GetOnlineFeaturesResponseMetadata.feature_names:type_name -> feast.serving.FeatureList + 14, // 8: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.timestamp:type_name -> google.protobuf.Timestamp + 10, // 9: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry + 15, // 10: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry.value:type_name -> feast.types.Value + 16, // 11: feast.serving.GetOnlineFeaturesRequest.EntitiesEntry.value:type_name -> feast.types.RepeatedValue + 16, // 12: feast.serving.GetOnlineFeaturesRequest.RequestContextEntry.value:type_name -> feast.types.RepeatedValue + 15, // 13: feast.serving.GetOnlineFeaturesResponse.FeatureVector.values:type_name -> feast.types.Value + 0, // 14: feast.serving.GetOnlineFeaturesResponse.FeatureVector.statuses:type_name -> feast.serving.FieldStatus + 14, // 15: feast.serving.GetOnlineFeaturesResponse.FeatureVector.event_timestamps:type_name -> google.protobuf.Timestamp + 1, // 16: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest + 6, // 17: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest + 2, // 18: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse + 7, // 19: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse + 18, // [18:20] is the sub-list for method output_type + 16, // [16:18] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_feast_serving_ServingService_proto_init() } @@ -1139,18 +998,6 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponseV2); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesResponseMetadata); i { case 0: return &v.state @@ -1162,7 +1009,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesRequestV2_EntityRow); i { case 0: return &v.state @@ -1174,20 +1021,8 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse_FieldValues); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponseV2_FeatureVector); i { + file_feast_serving_ServingService_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetOnlineFeaturesResponse_FeatureVector); i { case 0: return &v.state case 1: @@ -1209,7 +1044,7 @@ func file_feast_serving_ServingService_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_serving_ServingService_proto_rawDesc, NumEnums: 1, - NumMessages: 17, + NumMessages: 13, NumExtensions: 0, NumServices: 1, }, @@ -1239,7 +1074,7 @@ type ServingServiceClient interface { // Get information about this Feast serving. GetFeastServingInfo(ctx context.Context, in *GetFeastServingInfoRequest, opts ...grpc.CallOption) (*GetFeastServingInfoResponse, error) // Get online features synchronously. - GetOnlineFeatures(ctx context.Context, in *GetOnlineFeaturesRequest, opts ...grpc.CallOption) (*GetOnlineFeaturesResponseV2, error) + GetOnlineFeatures(ctx context.Context, in *GetOnlineFeaturesRequest, opts ...grpc.CallOption) (*GetOnlineFeaturesResponse, error) } type servingServiceClient struct { @@ -1259,8 +1094,8 @@ func (c *servingServiceClient) GetFeastServingInfo(ctx context.Context, in *GetF return out, nil } -func (c *servingServiceClient) GetOnlineFeatures(ctx context.Context, in *GetOnlineFeaturesRequest, opts ...grpc.CallOption) (*GetOnlineFeaturesResponseV2, error) { - out := new(GetOnlineFeaturesResponseV2) +func (c *servingServiceClient) GetOnlineFeatures(ctx context.Context, in *GetOnlineFeaturesRequest, opts ...grpc.CallOption) (*GetOnlineFeaturesResponse, error) { + out := new(GetOnlineFeaturesResponse) err := c.cc.Invoke(ctx, "/feast.serving.ServingService/GetOnlineFeatures", in, out, opts...) if err != nil { return nil, err @@ -1273,7 +1108,7 @@ type ServingServiceServer interface { // Get information about this Feast serving. GetFeastServingInfo(context.Context, *GetFeastServingInfoRequest) (*GetFeastServingInfoResponse, error) // Get online features synchronously. - GetOnlineFeatures(context.Context, *GetOnlineFeaturesRequest) (*GetOnlineFeaturesResponseV2, error) + GetOnlineFeatures(context.Context, *GetOnlineFeaturesRequest) (*GetOnlineFeaturesResponse, error) } // UnimplementedServingServiceServer can be embedded to have forward compatible implementations. @@ -1283,7 +1118,7 @@ type UnimplementedServingServiceServer struct { func (*UnimplementedServingServiceServer) GetFeastServingInfo(context.Context, *GetFeastServingInfoRequest) (*GetFeastServingInfoResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetFeastServingInfo not implemented") } -func (*UnimplementedServingServiceServer) GetOnlineFeatures(context.Context, *GetOnlineFeaturesRequest) (*GetOnlineFeaturesResponseV2, error) { +func (*UnimplementedServingServiceServer) GetOnlineFeatures(context.Context, *GetOnlineFeaturesRequest) (*GetOnlineFeaturesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetOnlineFeatures not implemented") } diff --git a/sdk/go/response.go b/sdk/go/response.go index 49c8904ab7..cdb2cbee38 100644 --- a/sdk/go/response.go +++ b/sdk/go/response.go @@ -19,7 +19,7 @@ var ( // OnlineFeaturesResponse is a wrapper around serving.GetOnlineFeaturesResponse. type OnlineFeaturesResponse struct { - RawResponse *serving.GetOnlineFeaturesResponseV2 + RawResponse *serving.GetOnlineFeaturesResponse } // Rows retrieves the result of the request as a list of Rows. diff --git a/sdk/go/response_test.go b/sdk/go/response_test.go index e9a9bc1605..693faae7e4 100644 --- a/sdk/go/response_test.go +++ b/sdk/go/response_test.go @@ -9,8 +9,8 @@ import ( ) var response = OnlineFeaturesResponse{ - RawResponse: &serving.GetOnlineFeaturesResponseV2{ - Results: []*serving.GetOnlineFeaturesResponseV2_FeatureVector{ + RawResponse: &serving.GetOnlineFeaturesResponse{ + Results: []*serving.GetOnlineFeaturesResponse_FeatureVector{ { Values: []*types.Value{Int64Val(1), Int64Val(2)}, Statuses: []serving.FieldStatus{ diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 64bf23ebde..438bfef6d2 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -33,6 +33,7 @@ import pandas as pd from colorama import Fore, Style +from google.protobuf.timestamp_pb2 import Timestamp from tqdm import tqdm from feast import feature_server, flags, flags_helper, utils @@ -1179,14 +1180,19 @@ def get_online_features( for k, v in join_key_python_values.items() } - # Populate result rows with join keys - result_rows = [ - GetOnlineFeaturesResponse.FieldValues() for _ in range(len(entity_rows)) - ] + # Populate online features response proto with join keys + online_features_response = GetOnlineFeaturesResponse( + results=[ + GetOnlineFeaturesResponse.FeatureVector() + for _ in range(len(entity_rows)) + ] + ) for key, values in join_key_proto_values.items(): - for row_idx, result_row in enumerate(result_rows): - result_row.fields[key].CopyFrom(values[row_idx]) - result_row.statuses[key] = FieldStatus.PRESENT + online_features_response.metadata.feature_names.val.append(key) + for row_idx, result_row in enumerate(online_features_response.results): + result_row.values.append(values[row_idx]) + result_row.statuses.append(FieldStatus.PRESENT) + result_row.event_timestamps.append(Timestamp()) # Initialize the set of EntityKeyProtos once and reuse them for each FeatureView # to avoid initialization overhead. @@ -1204,30 +1210,30 @@ def get_online_features( # Populate the result_rows with the Features from the OnlineStore inplace. self._populate_result_rows_from_feature_view( + online_features_response, entity_keys, full_feature_names, provider, requested_features, - result_rows, table, ) self._populate_request_data_features( - request_data_features, result_rows, + online_features_response, request_data_features ) if grouped_odfv_refs: self._augment_response_with_on_demand_transforms( + online_features_response, _feature_refs, requested_on_demand_feature_views, full_feature_names, - result_rows, ) self._drop_unneeded_columns( - requested_result_row_names, result_rows, + online_features_response, requested_result_row_names ) - return OnlineResponse(GetOnlineFeaturesResponse(field_values=result_rows)) + return OnlineResponse(online_features_response) @staticmethod def _get_table_entity_values( @@ -1270,8 +1276,8 @@ def _set_table_entity_keys( @staticmethod def _populate_request_data_features( + online_features_response: GetOnlineFeaturesResponse, request_data_features: Dict[str, List[Any]], - result_rows: List[GetOnlineFeaturesResponse.FieldValues], ): # Add more feature values to the existing result rows for the request data features for feature_name, feature_values in request_data_features.items(): @@ -1279,10 +1285,13 @@ def _populate_request_data_features( feature_values, ValueType.UNKNOWN ) + online_features_response.metadata.feature_names.val.append(feature_name) + for row_idx, proto_value in enumerate(proto_values): - result_row = result_rows[row_idx] - result_row.fields[feature_name].CopyFrom(proto_value) - result_row.statuses[feature_name] = FieldStatus.PRESENT + result_row = online_features_response.results[row_idx] + result_row.values.append(proto_value) + result_row.statuses.append(FieldStatus.PRESENT) + result_row.event_timestamps.append(Timestamp()) @staticmethod def get_needed_request_data( @@ -1321,11 +1330,11 @@ def ensure_request_data_values_exist( def _populate_result_rows_from_feature_view( self, + online_features_response: GetOnlineFeaturesResponse, entity_keys: List[EntityKeyProto], full_feature_names: bool, provider: Provider, requested_features: List[str], - result_rows: List[GetOnlineFeaturesResponse.FieldValues], table: FeatureView, ): read_rows = provider.online_read( @@ -1334,47 +1343,54 @@ def _populate_result_rows_from_feature_view( entity_keys=entity_keys, requested_features=requested_features, ) + requested_feature_refs = [ + f"{table.projection.name_to_use()}__{feature_name}" + if full_feature_names + else feature_name + for feature_name in requested_features + ] + online_features_response.metadata.feature_names.val.extend( + requested_feature_refs + ) # Each row is a set of features for a given entity key for row_idx, read_row in enumerate(read_rows): row_ts, feature_data = read_row - result_row = result_rows[row_idx] + result_row = online_features_response.results[row_idx] + row_ts_proto = Timestamp() + if row_ts is not None: + row_ts_proto.FromDatetime(row_ts) + result_row.event_timestamps.extend([row_ts_proto] * len(requested_features)) if feature_data is None: - for feature_name in requested_features: - feature_ref = ( - f"{table.projection.name_to_use()}__{feature_name}" - if full_feature_names - else feature_name - ) - result_row.statuses[feature_ref] = FieldStatus.NOT_FOUND + result_row.statuses.extend( + [FieldStatus.NOT_FOUND] * len(requested_features) + ) + result_row.values.extend([Value()] * len(requested_features)) else: - for feature_name in feature_data: - feature_ref = ( - f"{table.projection.name_to_use()}__{feature_name}" - if full_feature_names - else feature_name - ) - if feature_name in requested_features: - result_row.fields[feature_ref].CopyFrom( - feature_data[feature_name] - ) - result_row.statuses[feature_ref] = FieldStatus.PRESENT + for feature_name in requested_features: + if feature_name not in feature_data: + result_row.statuses.append(FieldStatus.NOT_FOUND) + result_row.values.append(Value()) + else: + result_row.statuses.append(FieldStatus.PRESENT) + result_row.values.append(feature_data[feature_name]) @staticmethod def _augment_response_with_on_demand_transforms( + online_features_response: GetOnlineFeaturesResponse, feature_refs: List[str], requested_on_demand_feature_views: List[OnDemandFeatureView], full_feature_names: bool, - result_rows: List[GetOnlineFeaturesResponse.FieldValues], ): """Computes on demand feature values and adds them to the result rows. - Assumes that 'result_rows' already contains the necessary request data and input feature - views for the on demand feature views. + Assumes that 'online_features_response' already contains the necessary request data and input feature + views for the on demand feature views. Unneeded feature values such as request data and + unrequested input feature views will be removed from 'online_features_response'. Args: + online_features_response: Protobuf object to populate feature_refs: List of all feature references to be returned. - requested_on_demand_feature_views: List of all odfvs that have been requested. full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to @@ -1396,9 +1412,7 @@ def _augment_response_with_on_demand_transforms( else feature_name ) - initial_response = OnlineResponse( - GetOnlineFeaturesResponse(field_values=result_rows) - ) + initial_response = OnlineResponse(online_features_response) initial_response_df = initial_response.to_df() # Apply on demand transformations and augment the result rows @@ -1412,48 +1426,56 @@ def _augment_response_with_on_demand_transforms( f for f in transformed_features_df.columns if f in _feature_refs ] - proto_values_by_column = { - feature: python_values_to_proto_values( + proto_values = [ + python_values_to_proto_values( transformed_features_df[feature].values, ValueType.UNKNOWN ) for feature in selected_subset - } + ] - for row_idx in range(len(result_rows)): - result_row = result_rows[row_idx] + odfv_result_names |= set(selected_subset) - for transformed_feature in selected_subset: - odfv_result_names.add(transformed_feature) - result_row.fields[transformed_feature].CopyFrom( - proto_values_by_column[transformed_feature][row_idx] - ) - result_row.statuses[transformed_feature] = FieldStatus.PRESENT + online_features_response.metadata.feature_names.val.extend(selected_subset) + + for row_idx in range(len(online_features_response.results)): + result_row = online_features_response.results[row_idx] + for feature_idx, transformed_feature in enumerate(selected_subset): + result_row.values.append(proto_values[feature_idx][row_idx]) + result_row.statuses.append(FieldStatus.PRESENT) + result_row.event_timestamps.append(Timestamp()) @staticmethod def _drop_unneeded_columns( + online_features_response: GetOnlineFeaturesResponse, requested_result_row_names: Set[str], - result_rows: List[GetOnlineFeaturesResponse.FieldValues], ): """ Unneeded feature values such as request data and unrequested input feature views will - be removed from 'result_rows'. + be removed from 'online_features_response'. Args: + online_features_response: Protobuf object to populate requested_result_row_names: Fields from 'result_rows' that have been requested, and therefore should not be dropped. - result_rows: List of result rows to be editted inplace. """ # Drop values that aren't needed - unneeded_features = [ - val - for val in result_rows[0].fields + unneeded_feature_indices = [ + idx + for idx, val in enumerate( + online_features_response.metadata.feature_names.val + ) if val not in requested_result_row_names ] - for row_idx in range(len(result_rows)): - result_row = result_rows[row_idx] - for unneeded_feature in unneeded_features: - result_row.fields.pop(unneeded_feature) - result_row.statuses.pop(unneeded_feature) + + for idx in reversed(unneeded_feature_indices): + del online_features_response.metadata.feature_names.val[idx] + + for row_idx in range(len(online_features_response.results)): + result_row = online_features_response.results[row_idx] + for idx in reversed(unneeded_feature_indices): + del result_row.values[idx] + del result_row.statuses[idx] + del result_row.event_timestamps[idx] def _get_feature_views_to_use( self, diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index b7f8680e1f..46592bf2a3 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -173,7 +173,7 @@ def online_read( val = ValueProto() val.ParseFromString(value_bin.value) res[feature_name] = val - result.append((value["event_ts"], res)) + result.append((datetime.fromisoformat(value["event_ts"]), res)) else: result.append((None, None)) return result diff --git a/sdk/python/feast/online_response.py b/sdk/python/feast/online_response.py index e6bf6be42c..bb69c6b9d9 100644 --- a/sdk/python/feast/online_response.py +++ b/sdk/python/feast/online_response.py @@ -18,6 +18,7 @@ from feast.feature_view import DUMMY_ENTITY_ID from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesResponse +from feast.type_map import feast_value_type_to_python_type class OnlineResponse: @@ -34,53 +35,30 @@ def __init__(self, online_response_proto: GetOnlineFeaturesResponse): """ self.proto = online_response_proto # Delete DUMMY_ENTITY_ID from proto if it exists - for item in self.proto.field_values: - if DUMMY_ENTITY_ID in item.statuses: - del item.statuses[DUMMY_ENTITY_ID] - if DUMMY_ENTITY_ID in item.fields: - del item.fields[DUMMY_ENTITY_ID] - - @property - def field_values(self): - """ - Getter for GetOnlineResponse's field_values. - """ - return self.proto.field_values + for idx, val in enumerate(self.proto.metadata.feature_names.val): + if val == DUMMY_ENTITY_ID: + del self.proto.metadata.feature_names.val[idx] + for result in self.proto.results: + del result.values[idx] + del result.statuses[idx] + del result.event_timestamps[idx] + break def to_dict(self) -> Dict[str, Any]: """ Converts GetOnlineFeaturesResponse features into a dictionary form. """ - # Status for every Feature should be present in every record. - features_dict: Dict[str, List[Any]] = { - k: list() for k in self.field_values[0].statuses.keys() - } - rows = [record.fields for record in self.field_values] - - # Find the first non-null instance of each Feature to determine - # which ValueType. - val_types = {k: None for k in features_dict.keys()} - for feature in features_dict.keys(): - for row in rows: - try: - val_types[feature] = row[feature].WhichOneof("val") - except KeyError: - continue - if val_types[feature] is not None: - break + response: Dict[str, List[Any]] = {} - # Now we know what attribute to fetch. - for feature, val_type in val_types.items(): - if val_type is None: - features_dict[feature] = [None] * len(rows) - else: - for row in rows: - val = getattr(row[feature], val_type) - if "_list_" in val_type: - val = list(val.val) - features_dict[feature].append(val) + for result in self.proto.results: + for idx, feature_ref in enumerate(self.proto.metadata.feature_names.val): + native_type_value = feast_value_type_to_python_type(result.values[idx]) + if feature_ref not in response: + response[feature_ref] = [native_type_value] + else: + response[feature_ref].append(native_type_value) - return features_dict + return response def to_df(self) -> pd.DataFrame: """ diff --git a/sdk/python/tests/integration/online_store/test_e2e_local.py b/sdk/python/tests/integration/online_store/test_e2e_local.py index dd900e90dc..7990227344 100644 --- a/sdk/python/tests/integration/online_store/test_e2e_local.py +++ b/sdk/python/tests/integration/online_store/test_e2e_local.py @@ -40,7 +40,14 @@ def _assert_online_features( # Float features should still be floats from the online store... assert ( - response.field_values[0].fields["driver_hourly_stats__conv_rate"].float_val > 0 + response.proto.results[0] + .values[ + list(response.proto.metadata.feature_names.val).index( + "driver_hourly_stats__conv_rate" + ) + ] + .float_val + > 0 ) result = response.to_dict() diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index c47f2bbfd0..1f1b619cd0 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -212,11 +212,11 @@ def _get_online_features_dict_remotely( time.sleep(1) else: raise Exception("Failed to get online features from remote feature server") - keys = response["field_values"][0]["statuses"].keys() + keys = response["metadata"]["feature_names"] # Get rid of unnecessary structure in the response, leaving list of dicts - response = [row["fields"] for row in response["field_values"]] + response = [row["values"] for row in response["results"]] # Convert list of dicts (response) into dict of lists which is the format of the return value - return {key: [row.get(key) for row in response] for key in keys} + return {key: [row[idx] for row in response] for idx, key in enumerate(keys)} def get_online_features_dict( diff --git a/sdk/python/tests/unit/test_proto_json.py b/sdk/python/tests/unit/test_proto_json.py index 1b352ccb19..6bfdbbbf91 100644 --- a/sdk/python/tests/unit/test_proto_json.py +++ b/sdk/python/tests/unit/test_proto_json.py @@ -9,7 +9,7 @@ ) from feast.protos.feast.types.Value_pb2 import RepeatedValue -FieldValues = GetOnlineFeaturesResponse.FieldValues +FeatureVector = GetOnlineFeaturesResponse.FeatureVector @pytest.fixture(scope="module") @@ -17,70 +17,63 @@ def proto_json_patch(): proto_json.patch() -def test_feast_value(proto_json_patch): - # FieldValues contains "map fields" proto field. +def test_feature_vector_values(proto_json_patch): + # FeatureVector contains "repeated values" proto field. # We want to test that feast.types.Value can take different types in JSON # without using additional structure (e.g. 1 instead of {int64_val: 1}). - field_values_str = """{ - "fields": { - "a": 1, - "b": 2.0, - "c": true, - "d": "foo", - "e": [1, 2, 3], - "f": [2.0, 3.0, 4.0, null], - "g": [true, false, true], - "h": ["foo", "bar", "foobar"], - "i": null - } + feature_vector_str = """{ + "values": [ + 1, + 2.0, + true, + "foo", + [1, 2, 3], + [2.0, 3.0, 4.0, null], + [true, false, true], + ["foo", "bar", "foobar"] + ] }""" - field_values_proto = FieldValues() - Parse(field_values_str, field_values_proto) - assertpy.assert_that(field_values_proto.fields.keys()).is_equal_to( - {"a", "b", "c", "d", "e", "f", "g", "h", "i"} - ) - assertpy.assert_that(field_values_proto.fields["a"].int64_val).is_equal_to(1) - assertpy.assert_that(field_values_proto.fields["b"].double_val).is_equal_to(2.0) - assertpy.assert_that(field_values_proto.fields["c"].bool_val).is_equal_to(True) - assertpy.assert_that(field_values_proto.fields["d"].string_val).is_equal_to("foo") - assertpy.assert_that(field_values_proto.fields["e"].int64_list_val.val).is_equal_to( + feature_vector_proto = FeatureVector() + Parse(feature_vector_str, feature_vector_proto) + assertpy.assert_that(len(feature_vector_proto.values)).is_equal_to(8) + assertpy.assert_that(feature_vector_proto.values[0].int64_val).is_equal_to(1) + assertpy.assert_that(feature_vector_proto.values[1].double_val).is_equal_to(2.0) + assertpy.assert_that(feature_vector_proto.values[2].bool_val).is_equal_to(True) + assertpy.assert_that(feature_vector_proto.values[3].string_val).is_equal_to("foo") + assertpy.assert_that(feature_vector_proto.values[4].int64_list_val.val).is_equal_to( [1, 2, 3] ) # Can't directly check equality to [2.0, 3.0, 4.0, float("nan")], because float("nan") != float("nan") assertpy.assert_that( - field_values_proto.fields["f"].double_list_val.val[:3] + feature_vector_proto.values[5].double_list_val.val[:3] ).is_equal_to([2.0, 3.0, 4.0]) - assertpy.assert_that(field_values_proto.fields["f"].double_list_val.val[3]).is_nan() - assertpy.assert_that(field_values_proto.fields["g"].bool_list_val.val).is_equal_to( + assertpy.assert_that(feature_vector_proto.values[5].double_list_val.val[3]).is_nan() + assertpy.assert_that(feature_vector_proto.values[6].bool_list_val.val).is_equal_to( [True, False, True] ) assertpy.assert_that( - field_values_proto.fields["h"].string_list_val.val + feature_vector_proto.values[7].string_list_val.val ).is_equal_to(["foo", "bar", "foobar"]) - assertpy.assert_that(field_values_proto.fields["i"].null_val).is_equal_to(0) # Now convert protobuf back to json and check that - field_values_json = MessageToDict(field_values_proto) - assertpy.assert_that(field_values_json["fields"].keys()).is_equal_to( - {"a", "b", "c", "d", "e", "f", "g", "h", "i"} - ) - assertpy.assert_that(field_values_json["fields"]["a"]).is_equal_to(1) - assertpy.assert_that(field_values_json["fields"]["b"]).is_equal_to(2.0) - assertpy.assert_that(field_values_json["fields"]["c"]).is_equal_to(True) - assertpy.assert_that(field_values_json["fields"]["d"]).is_equal_to("foo") - assertpy.assert_that(field_values_json["fields"]["e"]).is_equal_to([1, 2, 3]) + feature_vector_json = MessageToDict(feature_vector_proto) + assertpy.assert_that(len(feature_vector_json["values"])).is_equal_to(8) + assertpy.assert_that(feature_vector_json["values"][0]).is_equal_to(1) + assertpy.assert_that(feature_vector_json["values"][1]).is_equal_to(2.0) + assertpy.assert_that(feature_vector_json["values"][2]).is_equal_to(True) + assertpy.assert_that(feature_vector_json["values"][3]).is_equal_to("foo") + assertpy.assert_that(feature_vector_json["values"][4]).is_equal_to([1, 2, 3]) # Can't directly check equality to [2.0, 3.0, 4.0, float("nan")], because float("nan") != float("nan") - assertpy.assert_that(field_values_json["fields"]["f"][:3]).is_equal_to( + assertpy.assert_that(feature_vector_json["values"][5][:3]).is_equal_to( [2.0, 3.0, 4.0] ) - assertpy.assert_that(field_values_json["fields"]["f"][3]).is_nan() - assertpy.assert_that(field_values_json["fields"]["g"]).is_equal_to( + assertpy.assert_that(feature_vector_json["values"][5][3]).is_nan() + assertpy.assert_that(feature_vector_json["values"][6]).is_equal_to( [True, False, True] ) - assertpy.assert_that(field_values_json["fields"]["h"]).is_equal_to( + assertpy.assert_that(feature_vector_json["values"][7]).is_equal_to( ["foo", "bar", "foobar"] ) - assertpy.assert_that(field_values_json["fields"]["i"]).is_equal_to(None) def test_feast_repeated_value(proto_json_patch): From f32b4f45d534372720f44e238590fccbb9aa2fd8 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Tue, 18 Jan 2022 10:12:32 -0500 Subject: [PATCH 13/15] Adding a local feature server test (#2217) * merge Signed-off-by: Danny Chiao * Fix port collision Signed-off-by: Danny Chiao * Fix lint Signed-off-by: Danny Chiao * fix lint Signed-off-by: Danny Chiao * fix lint Signed-off-by: Danny Chiao * fix lint Signed-off-by: Danny Chiao * fix lint Signed-off-by: Danny Chiao * change static method Signed-off-by: Danny Chiao --- sdk/python/feast/feature_server.py | 4 ++- sdk/python/tests/conftest.py | 29 ++++++++++++++++--- ..._repo_with_duplicated_featureview_names.py | 4 +-- .../feature_repos/repo_configuration.py | 22 +++++++++++++- .../feature_repos/universal/feature_views.py | 4 +-- .../online_store/test_universal_online.py | 10 +++++-- 6 files changed, 60 insertions(+), 13 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index b813af1c63..010e6b58fc 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -1,3 +1,5 @@ +import traceback + import click import uvicorn from fastapi import FastAPI, HTTPException, Request @@ -59,7 +61,7 @@ def get_online_features(body=Depends(get_body)): ) except Exception as e: # Print the original exception on the server side - logger.exception(e) + logger.exception(traceback.format_exc()) # Raise HTTPException to return the error message to the client raise HTTPException(status_code=500, detail=str(e)) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 61e591f237..49f32379a3 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -13,7 +13,9 @@ # limitations under the License. import logging import multiprocessing +import time from datetime import datetime, timedelta +from multiprocessing import Process from sys import platform from typing import List @@ -21,6 +23,7 @@ import pytest from _pytest.nodes import Item +from feast import FeatureStore from tests.data.data_creator import create_dataset from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, @@ -137,23 +140,41 @@ def simple_dataset_2() -> pd.DataFrame: return pd.DataFrame.from_dict(data) +def start_test_local_server(repo_path: str, port: int): + fs = FeatureStore(repo_path) + fs.serve("localhost", port, no_access_log=True) + + @pytest.fixture( params=FULL_REPO_CONFIGS, scope="session", ids=[str(c) for c in FULL_REPO_CONFIGS] ) -def environment(request): - e = construct_test_environment(request.param) +def environment(request, worker_id: str): + e = construct_test_environment(request.param, worker_id=worker_id) + proc = Process( + target=start_test_local_server, + args=(e.feature_store.repo_path, e.get_local_server_port()), + daemon=True, + ) + if e.python_feature_server and e.test_repo_config.provider == "local": + proc.start() + # Wait for server to start + time.sleep(3) def cleanup(): e.feature_store.teardown() + if proc.is_alive(): + proc.kill() request.addfinalizer(cleanup) + return e @pytest.fixture() def local_redis_environment(request, worker_id): - - e = construct_test_environment(IntegrationTestRepoConfig(online_store=REDIS_CONFIG)) + e = construct_test_environment( + IntegrationTestRepoConfig(online_store=REDIS_CONFIG), worker_id=worker_id + ) def cleanup(): e.feature_store.teardown() diff --git a/sdk/python/tests/example_repos/example_feature_repo_with_duplicated_featureview_names.py b/sdk/python/tests/example_repos/example_feature_repo_with_duplicated_featureview_names.py index e4c7abed0f..84d57bf038 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_with_duplicated_featureview_names.py +++ b/sdk/python/tests/example_repos/example_feature_repo_with_duplicated_featureview_names.py @@ -10,7 +10,7 @@ name="driver_hourly_stats", # Intentionally use the same FeatureView name entities=["driver_id"], online=False, - input=driver_hourly_stats, + batch_source=driver_hourly_stats, ttl=Duration(seconds=10), tags={}, ) @@ -19,7 +19,7 @@ name="driver_hourly_stats", # Intentionally use the same FeatureView name entities=["driver_id"], online=False, - input=driver_hourly_stats, + batch_source=driver_hourly_stats, ttl=Duration(seconds=10), tags={}, ) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 6dedfb63b2..63ee4fe7bc 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -1,6 +1,7 @@ import importlib import json import os +import re import tempfile import uuid from dataclasses import dataclass, field @@ -51,6 +52,7 @@ DEFAULT_FULL_REPO_CONFIGS: List[IntegrationTestRepoConfig] = [ # Local configurations IntegrationTestRepoConfig(), + IntegrationTestRepoConfig(python_feature_server=True), ] if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True": DEFAULT_FULL_REPO_CONFIGS.extend( @@ -217,6 +219,7 @@ class Environment: feature_store: FeatureStore data_source_creator: DataSourceCreator python_feature_server: bool + worker_id: str end_date: datetime = field( default=datetime.utcnow().replace(microsecond=0, second=0, minute=0) @@ -225,6 +228,20 @@ class Environment: def __post_init__(self): self.start_date: datetime = self.end_date - timedelta(days=3) + def get_feature_server_endpoint(self) -> str: + if self.python_feature_server and self.test_repo_config.provider == "local": + return f"http://localhost:{self.get_local_server_port()}" + return self.feature_store.get_feature_server_endpoint() + + def get_local_server_port(self) -> int: + # Heuristic when running with xdist to extract unique ports for each worker + parsed_worker_id = re.findall("gw(\\d+)", self.worker_id) + if len(parsed_worker_id) != 0: + worker_id_num = int(parsed_worker_id[0]) + else: + worker_id_num = 0 + return 6566 + worker_id_num + def table_name_from_data_source(ds: DataSource) -> Optional[str]: if hasattr(ds, "table_ref"): @@ -237,6 +254,7 @@ def table_name_from_data_source(ds: DataSource) -> Optional[str]: def construct_test_environment( test_repo_config: IntegrationTestRepoConfig, test_suite_name: str = "integration_test", + worker_id: str = "worker_id", ) -> Environment: _uuid = str(uuid.uuid4()).replace("-", "")[:8] @@ -254,7 +272,7 @@ def construct_test_environment( repo_dir_name = tempfile.mkdtemp() - if test_repo_config.python_feature_server: + if test_repo_config.python_feature_server and test_repo_config.provider == "aws": from feast.infra.feature_servers.aws_lambda.config import ( AwsLambdaFeatureServerConfig, ) @@ -266,6 +284,7 @@ def construct_test_environment( registry = f"s3://feast-integration-tests/registries/{project}/registry.db" else: + # Note: even if it's a local feature server, the repo config does not have this configured feature_server = None registry = str(Path(repo_dir_name) / "registry.db") @@ -293,6 +312,7 @@ def construct_test_environment( feature_store=fs, data_source_creator=offline_creator, python_feature_server=test_repo_config.python_feature_server, + worker_id=worker_id, ) return environment diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 3d19212f48..f68add88cb 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -20,7 +20,7 @@ def driver_feature_view( entities=["driver"], features=None if infer_features else [Feature("value", value_type)], ttl=timedelta(days=5), - input=data_source, + batch_source=data_source, ) @@ -35,7 +35,7 @@ def global_feature_view( entities=[], features=None if infer_features else [Feature("entityless_value", value_type)], ttl=timedelta(days=5), - input=data_source, + batch_source=data_source, ) diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 1f1b619cd0..b23c68033e 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -185,7 +185,7 @@ def _get_online_features_dict_remotely( The output should be identical to: - >>> fs.get_online_features(features=features, entity_rows=entity_rows, full_feature_names=full_feature_names).to_dict() + fs.get_online_features(features=features, entity_rows=entity_rows, full_feature_names=full_feature_names).to_dict() This makes it easy to test the remote feature server by comparing the output to the local method. @@ -212,6 +212,10 @@ def _get_online_features_dict_remotely( time.sleep(1) else: raise Exception("Failed to get online features from remote feature server") + if "metadata" not in response: + raise Exception( + f"Failed to get online features from remote feature server {response}" + ) keys = response["metadata"]["feature_names"] # Get rid of unnecessary structure in the response, leaving list of dicts response = [row["values"] for row in response["results"]] @@ -238,8 +242,8 @@ def get_online_features_dict( assertpy.assert_that(online_features).is_not_none() dict1 = online_features.to_dict() - endpoint = environment.feature_store.get_feature_server_endpoint() - # If endpoint is None, it means that the remote feature server isn't configured + endpoint = environment.get_feature_server_endpoint() + # If endpoint is None, it means that a local / remote feature server aren't configured if endpoint is not None: dict2 = _get_online_features_dict_remotely( endpoint=endpoint, From 05f4e8f9df1b637cde412edb086a93ca86b56788 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Tue, 18 Jan 2022 16:18:32 +0000 Subject: [PATCH 14/15] Python FeatureServer optimization (#2202) * Optimize Python FeatureServer Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Handle `RepeatedValue` proto in `_get_online_features` Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Only initialize `Timestamp` once Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> * Don't use `defaultdict` Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- sdk/python/feast/feature_server.py | 16 +- sdk/python/feast/feature_store.py | 257 ++++++++++++++++++----------- 2 files changed, 163 insertions(+), 110 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 010e6b58fc..1f4513fa37 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -10,7 +10,6 @@ import feast from feast import proto_json from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest -from feast.type_map import feast_value_type_to_python_type def get_app(store: "feast.FeatureStore"): @@ -43,16 +42,11 @@ def get_online_features(body=Depends(get_body)): if any(batch_size != num_entities for batch_size in batch_sizes): raise HTTPException(status_code=500, detail="Uneven number of columns") - entity_rows = [ - { - k: feast_value_type_to_python_type(v.val[idx]) - for k, v in request_proto.entities.items() - } - for idx in range(num_entities) - ] - - response_proto = store.get_online_features( - features, entity_rows, full_feature_names=full_feature_names + response_proto = store._get_online_features( + features, + request_proto.entities, + full_feature_names=full_feature_names, + native_entity_values=False, ).proto # Convert the Protobuf object to JSON and return it diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 438bfef6d2..39273b56c2 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -23,8 +23,10 @@ Dict, Iterable, List, + Mapping, NamedTuple, Optional, + Sequence, Set, Tuple, Union, @@ -72,7 +74,7 @@ GetOnlineFeaturesResponse, ) from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.protos.feast.types.Value_pb2 import Value +from feast.protos.feast.types.Value_pb2 import RepeatedValue, Value from feast.registry import Registry from feast.repo_config import RepoConfig, load_repo_config from feast.request_feature_view import RequestFeatureView @@ -267,14 +269,18 @@ def _list_feature_views( return feature_views @log_exceptions_and_usage - def list_on_demand_feature_views(self) -> List[OnDemandFeatureView]: + def list_on_demand_feature_views( + self, allow_cache: bool = False + ) -> List[OnDemandFeatureView]: """ Retrieves the list of on demand feature views from the registry. Returns: A list of on demand feature views. """ - return self._registry.list_on_demand_feature_views(self.project) + return self._registry.list_on_demand_feature_views( + self.project, allow_cache=allow_cache + ) @log_exceptions_and_usage def get_entity(self, name: str) -> Entity: @@ -1067,6 +1073,30 @@ def get_online_features( ... ) >>> online_response_dict = online_response.to_dict() """ + columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} + for entity_row in entity_rows: + for key, value in entity_row.items(): + try: + columnar[key].append(value) + except KeyError as e: + raise ValueError("All entity_rows must have the same keys.") from e + + return self._get_online_features( + features=features, + entity_values=columnar, + full_feature_names=full_feature_names, + native_entity_values=True, + ) + + def _get_online_features( + self, + features: Union[List[str], FeatureService], + entity_values: Mapping[ + str, Union[Sequence[Any], Sequence[Value], RepeatedValue] + ], + full_feature_names: bool = False, + native_entity_values: bool = True, + ): _feature_refs = self._get_features(features, allow_cache=True) ( requested_feature_views, @@ -1076,6 +1106,29 @@ def get_online_features( features=features, allow_cache=True, hide_dummy_entity=False ) + entity_name_to_join_key_map, entity_type_map = self._get_entity_maps( + requested_feature_views + ) + + # Extract Sequence from RepeatedValue Protobuf. + entity_value_lists: Dict[str, Union[List[Any], List[Value]]] = { + k: list(v) if isinstance(v, Sequence) else list(v.val) + for k, v in entity_values.items() + } + + entity_proto_values: Dict[str, List[Value]] + if native_entity_values: + # Convert values to Protobuf once. + entity_proto_values = { + k: python_values_to_proto_values( + v, entity_type_map.get(k, ValueType.UNKNOWN) + ) + for k, v in entity_value_lists.items() + } + else: + entity_proto_values = entity_value_lists + + num_rows = _validate_entity_values(entity_proto_values) _validate_feature_refs(_feature_refs, full_feature_names) ( grouped_refs, @@ -1101,111 +1154,72 @@ def get_online_features( } feature_views = list(view for view, _ in grouped_refs) - entityless_case = DUMMY_ENTITY_NAME in [ - entity_name - for feature_view in feature_views - for entity_name in feature_view.entities - ] - - provider = self._get_provider() - entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) - entity_name_to_join_key_map: Dict[str, str] = {} - join_key_to_entity_type_map: Dict[str, ValueType] = {} - for entity in entities: - entity_name_to_join_key_map[entity.name] = entity.join_key - join_key_to_entity_type_map[entity.join_key] = entity.value_type - for feature_view in requested_feature_views: - for entity_name in feature_view.entities: - entity = self._registry.get_entity( - entity_name, self.project, allow_cache=True - ) - # User directly uses join_key as the entity reference in the entity_rows for the - # entity mapping case. - entity_name = feature_view.projection.join_key_map.get( - entity.join_key, entity.name - ) - join_key = feature_view.projection.join_key_map.get( - entity.join_key, entity.join_key - ) - entity_name_to_join_key_map[entity_name] = join_key - join_key_to_entity_type_map[join_key] = entity.value_type needed_request_data, needed_request_fv_features = self.get_needed_request_data( grouped_odfv_refs, grouped_request_fv_refs ) - join_key_rows = [] - request_data_features: Dict[str, List[Any]] = defaultdict(list) + join_key_values: Dict[str, List[Value]] = {} + request_data_features: Dict[str, List[Value]] = {} # Entity rows may be either entities or request data. - for row in entity_rows: - join_key_row = {} - for entity_name, entity_value in row.items(): - # Found request data - if ( - entity_name in needed_request_data - or entity_name in needed_request_fv_features - ): - if entity_name in needed_request_fv_features: - # If the data was requested as a feature then - # make sure it appears in the result. - requested_result_row_names.add(entity_name) - request_data_features[entity_name].append(entity_value) - else: - try: - join_key = entity_name_to_join_key_map[entity_name] - except KeyError: - raise EntityNotFoundException(entity_name, self.project) - # All join keys should be returned in the result. - requested_result_row_names.add(join_key) - join_key_row[join_key] = entity_value - if entityless_case: - join_key_row[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL - if len(join_key_row) > 0: - # May be empty if this entity row was request data - join_key_rows.append(join_key_row) + for entity_name, values in entity_proto_values.items(): + # Found request data + if ( + entity_name in needed_request_data + or entity_name in needed_request_fv_features + ): + if entity_name in needed_request_fv_features: + # If the data was requested as a feature then + # make sure it appears in the result. + requested_result_row_names.add(entity_name) + request_data_features[entity_name] = values + else: + try: + join_key = entity_name_to_join_key_map[entity_name] + except KeyError: + raise EntityNotFoundException(entity_name, self.project) + # All join keys should be returned in the result. + requested_result_row_names.add(join_key) + join_key_values[join_key] = values self.ensure_request_data_values_exist( needed_request_data, needed_request_fv_features, request_data_features ) - # Convert join_key_rows from rowise to columnar. - join_key_python_values: Dict[str, List[Value]] = defaultdict(list) - for join_key_row in join_key_rows: - for join_key, value in join_key_row.items(): - join_key_python_values[join_key].append(value) - - # Convert all join key values to Protobuf Values - join_key_proto_values = { - k: python_values_to_proto_values(v, join_key_to_entity_type_map[k]) - for k, v in join_key_python_values.items() - } - - # Populate online features response proto with join keys + # Populate online features response proto with join keys and request data features online_features_response = GetOnlineFeaturesResponse( - results=[ - GetOnlineFeaturesResponse.FeatureVector() - for _ in range(len(entity_rows)) - ] + results=[GetOnlineFeaturesResponse.FeatureVector() for _ in range(num_rows)] ) - for key, values in join_key_proto_values.items(): - online_features_response.metadata.feature_names.val.append(key) - for row_idx, result_row in enumerate(online_features_response.results): - result_row.values.append(values[row_idx]) - result_row.statuses.append(FieldStatus.PRESENT) - result_row.event_timestamps.append(Timestamp()) + self._populate_result_rows_from_columnar( + online_features_response=online_features_response, + data=dict(**join_key_values, **request_data_features), + ) + + # Add the Entityless case after populating result rows to avoid having to remove + # it later. + entityless_case = DUMMY_ENTITY_NAME in [ + entity_name + for feature_view in feature_views + for entity_name in feature_view.entities + ] + if entityless_case: + join_key_values[DUMMY_ENTITY_ID] = python_values_to_proto_values( + [DUMMY_ENTITY_VAL] * num_rows, DUMMY_ENTITY.value_type + ) # Initialize the set of EntityKeyProtos once and reuse them for each FeatureView # to avoid initialization overhead. - entity_keys = [EntityKeyProto() for _ in range(len(join_key_rows))] + entity_keys = [EntityKeyProto() for _ in range(num_rows)] + provider = self._get_provider() for table, requested_features in grouped_refs: # Get the correct set of entity values with the correct join keys. - entity_values = self._get_table_entity_values( - table, entity_name_to_join_key_map, join_key_proto_values, + table_entity_values = self._get_table_entity_values( + table, entity_name_to_join_key_map, join_key_values, ) # Set the EntityKeyProtos inplace. self._set_table_entity_keys( - entity_values, entity_keys, + table_entity_values, entity_keys, ) # Populate the result_rows with the Features from the OnlineStore inplace. @@ -1218,10 +1232,6 @@ def get_online_features( table, ) - self._populate_request_data_features( - online_features_response, request_data_features - ) - if grouped_odfv_refs: self._augment_response_with_on_demand_transforms( online_features_response, @@ -1235,6 +1245,50 @@ def get_online_features( ) return OnlineResponse(online_features_response) + @staticmethod + def _get_columnar_entity_values( + rowise: Optional[List[Dict[str, Any]]], columnar: Optional[Dict[str, List[Any]]] + ) -> Dict[str, List[Any]]: + if (rowise is None and columnar is None) or ( + rowise is not None and columnar is not None + ): + raise ValueError( + "Exactly one of `columnar_entity_values` and `rowise_entity_values` must be set." + ) + + if rowise is not None: + # Convert entity_rows from rowise to columnar. + res = defaultdict(list) + for entity_row in rowise: + for key, value in entity_row.items(): + res[key].append(value) + return res + return cast(Dict[str, List[Any]], columnar) + + def _get_entity_maps(self, feature_views): + entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) + entity_name_to_join_key_map: Dict[str, str] = {} + entity_type_map: Dict[str, ValueType] = {} + for entity in entities: + entity_name_to_join_key_map[entity.name] = entity.join_key + entity_type_map[entity.name] = entity.value_type + for feature_view in feature_views: + for entity_name in feature_view.entities: + entity = self._registry.get_entity( + entity_name, self.project, allow_cache=True + ) + # User directly uses join_key as the entity reference in the entity_rows for the + # entity mapping case. + entity_name = feature_view.projection.join_key_map.get( + entity.join_key, entity.name + ) + join_key = feature_view.projection.join_key_map.get( + entity.join_key, entity.join_key + ) + entity_name_to_join_key_map[entity_name] = join_key + entity_type_map[join_key] = entity.value_type + return entity_name_to_join_key_map, entity_type_map + @staticmethod def _get_table_entity_values( table: FeatureView, @@ -1275,23 +1329,21 @@ def _set_table_entity_keys( entity_key.entity_values.extend(next(rowise_values)) @staticmethod - def _populate_request_data_features( + def _populate_result_rows_from_columnar( online_features_response: GetOnlineFeaturesResponse, - request_data_features: Dict[str, List[Any]], + data: Dict[str, List[Value]], ): - # Add more feature values to the existing result rows for the request data features - for feature_name, feature_values in request_data_features.items(): - proto_values = python_values_to_proto_values( - feature_values, ValueType.UNKNOWN - ) + timestamp = Timestamp() # Only initialize this timestamp once. + # Add more values to the existing result rows + for feature_name, feature_values in data.items(): online_features_response.metadata.feature_names.val.append(feature_name) - for row_idx, proto_value in enumerate(proto_values): + for row_idx, proto_value in enumerate(feature_values): result_row = online_features_response.results[row_idx] result_row.values.append(proto_value) result_row.statuses.append(FieldStatus.PRESENT) - result_row.event_timestamps.append(Timestamp()) + result_row.event_timestamps.append(timestamp) @staticmethod def get_needed_request_data( @@ -1567,6 +1619,13 @@ def serve_transformations(self, port: int) -> None: transformation_server.start_server(self, port) +def _validate_entity_values(join_key_values: Dict[str, List[Value]]): + set_of_row_lengths = {len(v) for v in join_key_values.values()} + if len(set_of_row_lengths) > 1: + raise ValueError("All entity rows must have the same columns.") + return set_of_row_lengths.pop() + + def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): collided_feature_refs = [] From 1f3a595ea3879e8800cf1c290db7cdbac196164d Mon Sep 17 00:00:00 2001 From: pyalex Date: Wed, 19 Jan 2022 14:33:37 +0700 Subject: [PATCH 15/15] clean up .prow.yaml Signed-off-by: pyalex --- .prow.yaml | 126 ----------------------------------------------------- 1 file changed, 126 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index b03a71a475..4c8372cc7c 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -1,102 +1,4 @@ -presubmits: -- name: test-core-and-ingestion - decorate: true - spec: - containers: - - image: maven:3.6-jdk-11 - command: ["infra/scripts/test-java-core-ingestion.sh"] - resources: - requests: - cpu: "2000m" - memory: "1536Mi" - skip_branches: - - ^v0\.(3|4)-branch$ - -- name: test-core-and-ingestion-java-8 - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-8 - command: ["infra/scripts/test-java-core-ingestion.sh"] - resources: - requests: - cpu: "2000m" - memory: "1536Mi" - branches: - - ^v0\.(3|4)-branch$ - -- name: test-serving - decorate: true - spec: - containers: - - image: maven:3.6-jdk-11 - command: ["infra/scripts/test-java-serving.sh"] - skip_branches: - - ^v0\.(3|4)-branch$ - -- name: test-serving-java-8 - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-8 - command: ["infra/scripts/test-java-serving.sh"] - branches: - - ^v0\.(3|4)-branch$ - -- name: test-java-sdk - decorate: true - spec: - containers: - - image: maven:3.6-jdk-11 - command: ["infra/scripts/test-java-sdk.sh"] - skip_branches: - - ^v0\.(3|4)-branch$ - -- name: test-java-sdk-java-8 - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-8 - command: ["infra/scripts/test-java-sdk.sh"] - branches: - - ^v0\.(3|4)-branch$ - -- name: test-golang-sdk - decorate: true - spec: - containers: - - image: golang:1.13 - command: ["infra/scripts/test-golang-sdk.sh"] - postsubmits: -- name: publish-python-sdk - decorate: true - spec: - containers: - - image: python:3 - command: - - sh - - -c - - | - make package-protos && make compile-protos-python && infra/scripts/publish-python-sdk.sh \ - --directory-path sdk/python --repository pypi - volumeMounts: - - name: pypirc - mountPath: /root/.pypirc - subPath: .pypirc - readOnly: true - volumes: - - name: pypirc - secret: - secretName: pypirc - branches: - # Filter on tags with semantic versioning, prefixed with "v" - # https://github.com/semver/semver/issues/232 - - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ - - name: publish-java-sdk decorate: true spec: @@ -128,31 +30,3 @@ postsubmits: branches: # Filter on tags with semantic versioning, prefixed with "v". - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ - -- name: publish-java-8-sdk - decorate: true - spec: - containers: - - image: maven:3.6-jdk-8 - command: - - bash - - -c - - infra/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1} - volumeMounts: - - name: gpg-keys - mountPath: /etc/gpg - readOnly: true - - name: maven-settings - mountPath: /root/.m2/settings.xml - subPath: settings.xml - readOnly: true - volumes: - - name: gpg-keys - secret: - secretName: gpg-keys - - name: maven-settings - secret: - secretName: maven-settings - branches: - # Filter on tags with semantic versioning, prefixed with "v". v0.3 and v0.4 only. - - ^v0\.(3|4)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$