From e78f8332079afb4d0bab50b6145c3d19130c521b Mon Sep 17 00:00:00 2001 From: kennethmhc Date: Wed, 21 Feb 2024 17:34:41 +0100 Subject: [PATCH] [FSTORE-1134] Enable updating embedding (#1495) (cherry picked from commit 2191bade16d5b4b10823c06740cbe33a8d182327) --- .../featuregroup/FeaturegroupService.java | 1 + .../featuregroup/FeaturegroupController.java | 29 +++++++++++++++++++ .../cached/CachedFeaturegroupController.java | 12 +++++--- .../stream/StreamFeatureGroupController.java | 12 +++++--- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java index 87c1882774..68bb56a5d0 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java @@ -332,6 +332,7 @@ public Response getFeatureGroupForOnlinefs( verifyIdProvided(featuregroupId); Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); FeaturegroupDTO featuregroupDTO = new FeaturegroupDTO(featuregroup); + featuregroupDTO.setFeatures(featuregroupController.getPrimaryKey(featuregroup)); GenericEntity featuregroupGeneric = new GenericEntity(featuregroupDTO) {}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featuregroupGeneric).build(); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java index d655a33b0b..2cfc21b233 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java @@ -381,6 +381,35 @@ public FeaturegroupDTO convertFeaturegrouptoDTO(Featuregroup featuregroup, Proje } } + public List getPrimaryKey(Featuregroup featuregroup) throws FeaturestoreException { + switch (featuregroup.getFeaturegroupType()) { + case CACHED_FEATURE_GROUP: + return cachedFeaturegroupController.getPrimaryKeys(featuregroup) + .stream().map(pk -> { + FeatureGroupFeatureDTO featureDTO = new FeatureGroupFeatureDTO(); + featureDTO.setName(pk); + featureDTO.setPrimary(true); + return featureDTO; + }).collect(Collectors.toList()); + case STREAM_FEATURE_GROUP: + return streamFeatureGroupController.getPrimaryKeys(featuregroup) + .stream().map(pk -> { + FeatureGroupFeatureDTO featureDTO = new FeatureGroupFeatureDTO(); + featureDTO.setName(pk); + featureDTO.setPrimary(true); + return featureDTO; + }).collect(Collectors.toList()); + case ON_DEMAND_FEATURE_GROUP: + return onDemandFeaturegroupController.getFeaturesDTO(featuregroup) + .stream().filter(FeatureGroupFeatureDTO::getPrimary).collect(Collectors.toList()); + default: + throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATUREGROUP_TYPE.getMessage() + + ", Recognized Feature group types are: " + FeaturegroupType.ON_DEMAND_FEATURE_GROUP + "," + + FeaturegroupType.STREAM_FEATURE_GROUP + ", and: " + FeaturegroupType.CACHED_FEATURE_GROUP + + ". The provided feature group type was not recognized: " + featuregroup.getFeaturegroupType()); + } + } + /** * Retrieves a list of feature groups with a specific name from a specific feature store * diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/CachedFeaturegroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/CachedFeaturegroupController.java index 74e31f5512..f5081d0891 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/CachedFeaturegroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/CachedFeaturegroupController.java @@ -210,10 +210,7 @@ public CachedFeaturegroupDTO convertCachedFeaturegroupToDTO(Featuregroup feature public List getFeaturesDTO(Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException { - Set primaryKeys = featuregroup.getCachedFeaturegroup().getFeaturesExtraConstraints().stream() - .filter(CachedFeatureExtraConstraints::getPrimary) - .map(CachedFeatureExtraConstraints::getName) - .collect(Collectors.toSet()); + Set primaryKeys = getPrimaryKeys(featuregroup); Set precombineKeys = featuregroup.getCachedFeaturegroup().getFeaturesExtraConstraints().stream() .filter(CachedFeatureExtraConstraints::getHudiPrecombineKey) @@ -242,6 +239,13 @@ public List getFeaturesDTO(Featuregroup featuregroup, Pr return featureGroupFeatures; } + public Set getPrimaryKeys(Featuregroup featuregroup) { + return featuregroup.getCachedFeaturegroup().getFeaturesExtraConstraints().stream() + .filter(CachedFeatureExtraConstraints::getPrimary) + .map(CachedFeatureExtraConstraints::getName) + .collect(Collectors.toSet()); + } + public List getFeaturesDTOOnlineChecked(Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/stream/StreamFeatureGroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/stream/StreamFeatureGroupController.java index 8c0039fc27..6154bc230c 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/stream/StreamFeatureGroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/stream/StreamFeatureGroupController.java @@ -101,10 +101,7 @@ public StreamFeatureGroupDTO convertStreamFeatureGroupToDTO(Featuregroup feature public List getFeaturesDTO(Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException { - Set primaryKeys = featuregroup.getStreamFeatureGroup().getFeaturesExtraConstraints().stream() - .filter(CachedFeatureExtraConstraints::getPrimary) - .map(CachedFeatureExtraConstraints::getName) - .collect(Collectors.toSet()); + Set primaryKeys = getPrimaryKeys(featuregroup); Set precombineKeys = featuregroup.getStreamFeatureGroup().getFeaturesExtraConstraints().stream() .filter(CachedFeatureExtraConstraints::getHudiPrecombineKey) @@ -131,6 +128,13 @@ public List getFeaturesDTO(Featuregroup featuregroup, return featureGroupFeatures; } + public Set getPrimaryKeys(Featuregroup featuregroup) { + return featuregroup.getStreamFeatureGroup().getFeaturesExtraConstraints().stream() + .filter(CachedFeatureExtraConstraints::getPrimary) + .map(CachedFeatureExtraConstraints::getName) + .collect(Collectors.toSet()); + } + public List getFeaturesDTOOnlineChecked(Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException {