diff --git a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb index a0e00e26e1..6800c23e4b 100644 --- a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb @@ -3235,6 +3235,20 @@ expect(json_body.all? { |fg| fg[:name] == "featuregroup1" }).to be true end + it "should be able to list all featuregroups of the project's featurestore filtered by latest_version" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + + # get fgs + get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=latest_version" + expect_status_details(200) + names = json_body.map { |o| "#{o[:name]}" } + + expect(json_body.length).to eq(3) + expect(json_body.length).to eq(names.uniq.size) + expect(json_body.any? { |fg| fg[:name] == "featuregroup1" && fg[:version] == 3}).to be true + end + it "should be able to list all featuregroups of the project's featurestore filtered by name and version" do project = get_project featurestore_id = get_featurestore_id(project.id) diff --git a/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb b/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb index 9ce97fcf02..fbf064a160 100644 --- a/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb @@ -648,6 +648,20 @@ expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true end + it "should be able to get a list of feature view filtered by latest_version" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + + # Get the list + get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featureview?filter_by=latest_version" + expect_status_details(200) + names = json_body[:items].map { |o| "#{o[:name]}" } + + expect(json_body[:items].length).to eq(2) + expect(json_body[:items].length).to eq(names.uniq.size) + expect(json_body[:items].any? { |fv| fv[:name] == "featureview2" && fv[:version] == 2}).to be true + end + it "should be able to get a list of feature view filtered by name and version" do project = get_project featurestore_id = get_featurestore_id(project.id) diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupBeanParam.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupBeanParam.java index 43caa4c532..6326e87c42 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupBeanParam.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupBeanParam.java @@ -28,10 +28,9 @@ public class FeatureGroupBeanParam { @QueryParam("filter_by") - @ApiParam(value = "ex. filter_by=expectations:exp1&filter_by=expectations:exp2", - allowableValues = - "filter_by=expectations:exp1", - allowMultiple = true) + @ApiParam(value = "filter_by=latest_version", + allowableValues = "filter_by=latest_version,filter_by=name:value,filter_by=version:value", + allowMultiple = true) private Set filter; @QueryParam("sort_by") diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FilterBy.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FilterBy.java index 49963f8dcf..fec6eb25e9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FilterBy.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FilterBy.java @@ -29,7 +29,7 @@ public FilterBy(String param) { this.filter = FeaturegroupFacade.Filters.valueOf(param.substring(0, param.indexOf(':')).toUpperCase()); this.param = param.substring(param.indexOf(':') + 1); } else { - this.filter = FeaturegroupFacade.Filters.valueOf(param); + this.filter = FeaturegroupFacade.Filters.valueOf(param.toUpperCase()); this.param = this.filter.getDefaultParam(); } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java index f57ef0d7d3..9c622f4493 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java @@ -253,7 +253,6 @@ private void setFilter(Set filters, Query q) private void setFilterQuery(FilterBy filter, Query q) { switch (Filters.valueOf(filter.getValue())) { case NAME: - case EXPECTATIONS: q.setParameter(filter.getField(), filter.getParam()); break; case VERSION: @@ -267,7 +266,11 @@ private void setFilterQuery(FilterBy filter, Query q) { public enum Filters { NAME("NAME", "fg.name = :name", "name", ""), VERSION("VERSION", "fg.version = :version", "version", ""), - EXPECTATIONS("EXPECTATIONS", "", "expectations", ""); + LATEST_VERSION("LATEST_VERSION", String.format("%1$s.version = ( " + + "SELECT MAX(%2$s.version) " + + "FROM Featuregroup %2$s " + + "WHERE %1$s.name = %2$s.name AND %1$s.featurestore = %2$s.featurestore " + + ") ", "fg", "fg2"), null, null); private final String value; private final String sql; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java index 9202ce7cce..2000b3d6a0 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java @@ -27,7 +27,6 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; import javax.persistence.TypedQuery; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,11 +50,6 @@ public List findAll() { } public List findByFeaturestore(Featurestore featurestore, QueryParam queryParam) { - Boolean latestVersion = false; - if (queryParam != null && queryParam.getFilters().removeIf(filter -> filter.toString().equals("LATEST_VERSION"))) { - latestVersion = true; - } - Map extraParam = new HashMap<>(); extraParam.put("featurestore", featurestore); String queryStr = buildQuery("SELECT fv FROM FeatureView fv ", @@ -63,12 +57,7 @@ public List findByFeaturestore(Featurestore featurestore, QueryPara queryParam != null ? queryParam.getSorts(): null, "fv.featurestore = :featurestore "); Query q = makeQuery(queryStr, queryParam, extraParam); - List results = q.getResultList(); - - if (latestVersion) { - results = retainLatestVersion(results); - } - return results; + return q.getResultList(); } public Long countByFeaturestore(Featurestore featurestore) { @@ -77,19 +66,6 @@ public Long countByFeaturestore(Featurestore featurestore) { .getSingleResult(); } - List retainLatestVersion(List featureViews) { - Map latestVersion = new HashMap<>(); - for (FeatureView featureView : featureViews) { - if (!latestVersion.containsKey(featureView.getName())) { - latestVersion.put(featureView.getName(), featureView); - } else if (latestVersion.containsKey(featureView.getName()) && - featureView.getVersion() > latestVersion.get(featureView.getName()).getVersion()) { - latestVersion.put(featureView.getName(), featureView); - } - } - return new ArrayList<>(latestVersion.values()); - } - public Optional findByIdAndFeatureStore(Integer id, Featurestore featureStore) { try { return Optional.of( @@ -191,7 +167,11 @@ protected EntityManager getEntityManager() { public enum Filters { NAME("NAME", String.format("%s.name = :%%s ", FeatureView.TABLE_NAME_ALIAS), "name", ""), VERSION("VERSION", String.format("%s.version = :%%s ", FeatureView.TABLE_NAME_ALIAS), "version", ""), - LATEST_VERSION("LATEST_VERSION", null, null, null); + LATEST_VERSION("LATEST_VERSION", String.format("%1$s.version = ( " + + "SELECT MAX(%2$s.version) " + + "FROM FeatureView %2$s " + + "WHERE %1$s.name = %2$s.name AND %1$s.featurestore = %2$s.featurestore " + + ") ", FeatureView.TABLE_NAME_ALIAS, "fv2"), null, null); private final String name; private final String sql; diff --git a/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacadeTest.java b/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacadeTest.java deleted file mode 100644 index f2ab28ceca..0000000000 --- a/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacadeTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This file is part of Hopsworks - * Copyright (C) 2021, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - */ - -package io.hops.hopsworks.common.featurestore.featureview; - -import com.google.common.collect.Lists; -import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.*; - -public class FeatureViewFacadeTest { - - private FeatureViewFacade target = new FeatureViewFacade(); - - @Test - public void testRetainLatestVersion() { - FeatureView featureView1_1 = createFeatureView("test1", 1); - FeatureView featureView1_2 = createFeatureView("test1", 2); - FeatureView featureView2_1 = createFeatureView("test2", 1); - FeatureView featureView2_2 = createFeatureView("test2", 2); - FeatureView featureView3_1 = createFeatureView("test3", 1); - List featureViews = Lists.newArrayList( - featureView1_1, featureView1_2, - featureView2_1, featureView2_2, - featureView3_1 - ); - List actual = target.retainLatestVersion(featureViews); - List expected = Lists.newArrayList( - featureView1_2, - featureView2_2, - featureView3_1 - ); - assertEquals(actual.size(), expected.size()); - assertTrue(actual.containsAll(expected)); - assertTrue(expected.containsAll(actual)); - } - - public FeatureView createFeatureView(String name, Integer version) { - FeatureView featureView = new FeatureView(); - featureView.setName(name); - featureView.setVersion(version); - return featureView; - } - -} \ No newline at end of file