diff --git a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb index baa0465a59..59ed64f17e 100644 --- a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb @@ -3180,19 +3180,19 @@ featurestore_id = get_featurestore_id(project.id) # create some feature groups - create_cached_featuregroup(project.id, featurestore_id) + create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:1) expect_status_details(201) - create_cached_featuregroup(project.id, featurestore_id) + create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:2) expect_status_details(201) - create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:1) + create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:3) expect_status_details(201) - create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:2) + create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup2", version:1) expect_status_details(201) - create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:3) + create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup3", version:1) expect_status_details(201) end @@ -3207,25 +3207,25 @@ get get_featuregroups_endpoint parsed_json = JSON.parse(response.body) expect_status_details(200) - old_count = parsed_json.length + old_count = parsed_json["items"].size # create fg - json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id) + json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup4", version:1) expect_status_details(201) # get fgs get get_featuregroups_endpoint parsed_json = JSON.parse(response.body) expect_status_details(200) - count = parsed_json.length + count = parsed_json["items"].size # ensure that the created fg is in returned fgs expect(count).to eq(old_count + 1) - expect(parsed_json[0].key?("id")).to be true - expect(parsed_json[0].key?("featurestoreName")).to be true - expect(parsed_json[0].key?("name")).to be true - expect(parsed_json[0]["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true - expect(parsed_json.any? { |fg| fg["name"] == featuregroup_name }).to be true + expect(parsed_json["items"][0].key?("id")).to be true + expect(parsed_json["items"][0].key?("featurestoreName")).to be true + expect(parsed_json["items"][0].key?("name")).to be true + expect(parsed_json["items"][0]["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true + expect(parsed_json["items"].any? { |fg| fg["name"] == featuregroup_name }).to be true end it "should be able to list all featuregroups of the project's featurestore sorted by name" do @@ -3235,10 +3235,10 @@ # get fgs get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc" expect_status_details(200) - names = json_body.map { |o| "#{o[:name]}" } + names = json_body[:items].map { |o| "#{o[:name]}" } sorted_names = names.sort_by(&:downcase) - expect(json_body.length).to eq(6) + expect(json_body[:items].size).to eq(6) expect(names).to eq(sorted_names) end @@ -3249,10 +3249,11 @@ # get fgs get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc,VERSION:asc" expect_status_details(200) - names_versions = json_body.map { |o| "#{o[:name]}_#{o[:version]}" } + names_versions = json_body[:items].map { |o| "#{o[:name]}_#{o[:version]}" } sorted_names_versions = names_versions.sort_by(&:downcase) - expect(json_body.length).to eq(6) + expect(json_body[:items].size).to eq(6) + expect(json_body[:count]).to eq(6) expect(names_versions).to eq(sorted_names_versions) end @@ -3264,8 +3265,22 @@ get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=NAME:featuregroup1" expect_status_details(200) - expect(json_body.length).to eq(3) - expect(json_body.all? { |fg| fg[:name] == "featuregroup1" }).to be true + expect(json_body[:items].size).to eq(3) + expect(json_body[:count]).to eq(3) + expect(json_body[:items].all? { |fg| fg[:name] == "featuregroup1" }).to be true + end + + it "should be able to list all featuregroups of the project's featurestore filtered by name like" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + + # get fgs + get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=NAME_LIKE:group1" + expect_status_details(200) + + expect(json_body[:items].size).to eq(3) + expect(json_body[:count]).to eq(3) + expect(json_body[:items].all? { |fg| fg[:name] == "featuregroup1" }).to be true end it "should be able to list all featuregroups of the project's featurestore filtered by latest_version" do @@ -3275,11 +3290,12 @@ # get fgs get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=latest_version" expect_status_details(200) - names = json_body.map { |o| "#{o[:name]}" } + names = json_body[:items].map { |o| "#{o[:name]}" } - expect(json_body.length).to eq(3) - expect(json_body.length).to eq(names.uniq.size) - expect(json_body.any? { |fg| fg[:name] == "featuregroup1" && fg[:version] == 3}).to be true + expect(json_body[:items].size).to eq(4) + expect(json_body[:count]).to eq(4) + expect(json_body[:items].size).to eq(names.uniq.size) + expect(json_body[:items].any? { |fg| fg[:name] == "featuregroup1" && fg[:version] == 3}).to be true end it "should be able to list all featuregroups of the project's featurestore filtered by name and version" do @@ -3290,9 +3306,9 @@ get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=NAME:featuregroup1&filter_by=VERSION:2" expect_status_details(200) - expect(json_body.length).to eq(1) - expect(json_body.all? { |fg| fg[:name] == "featuregroup1" }).to be true - expect(json_body.all? { |fg| fg[:version] == 2 }).to be true + expect(json_body[:items].size).to eq(1) + expect(json_body[:items].all? { |fg| fg[:name] == "featuregroup1" }).to be true + expect(json_body[:items].all? { |fg| fg[:version] == 2 }).to be true end it "should be able to list all featuregroups of the project's featurestore limit" do @@ -3303,7 +3319,8 @@ get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc&limit=2" expect_status_details(200) - expect(json_body.length).to eq(2) + expect(json_body[:items].size).to eq(2) + expect(json_body[:count]).to eq(6) end it "should be able to list all featuregroups of the project's featurestore offset" do @@ -3313,14 +3330,15 @@ # get fgs get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc" expect_status_details(200) - names = json_body.map { |o| "#{o[:name]}" } + names = json_body[:items].map { |o| "#{o[:name]}" } get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc&offset=1" expect_status_details(200) - names_with_offset = json_body.map { |o| "#{o[:name]}" } + names_with_offset = json_body[:items].map { |o| "#{o[:name]}" } - expect(json_body.length).to eq(5) - for i in 0..json_body.length-1 do + expect(json_body[:items].size).to eq(5) + expect(json_body[:count]).to eq(6) + for i in 0..json_body[:items].size-1 do expect(names[i+1]).to eq(names_with_offset[i]) end end diff --git a/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb b/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb index fbf064a160..6915ebbb3e 100644 --- a/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb @@ -591,6 +591,7 @@ expect_status_details(200) expect(parsed_json["items"].size).to eq(3) + expect(parsed_json["count"]).to eq(3) end it "should be able to get a list of feature view sorted by id" do @@ -604,6 +605,7 @@ sorted_ids = ids.sort expect(json_body[:items].length).to eq(3) + expect(json_body[:count]).to eq(3) expect(ids).to eq(ids.sort) expect(ids).not_to eq(ids.sort {|x, y| y <=> x}) end @@ -618,6 +620,7 @@ ids = json_body[:items].map { |o| o[:id] } expect(json_body[:items].length).to eq(3) + expect(json_body[:count]).to eq(3) expect(ids).to eq(ids.sort {|x, y| y <=> x}) expect(ids).not_to eq(ids.sort) end @@ -633,6 +636,7 @@ sorted_names_versions = names_versions.sort_by(&:downcase) expect(json_body[:items].length).to eq(3) + expect(json_body[:count]).to eq(3) expect(names_versions).to eq(sorted_names_versions) end @@ -645,6 +649,20 @@ expect_status_details(200) expect(json_body[:items].length).to eq(2) + expect(json_body[:count]).to eq(2) + expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true + end + + it "should be able to get a list of feature view filtered by name like" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + + # Get the list + get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featureview?filter_by=NAME_LIKE:view2" + expect_status_details(200) + + expect(json_body[:items].length).to eq(2) + expect(json_body[:count]).to eq(2) expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true end @@ -658,6 +676,7 @@ names = json_body[:items].map { |o| "#{o[:name]}" } expect(json_body[:items].length).to eq(2) + expect(json_body[:count]).to eq(2) expect(json_body[:items].length).to eq(names.uniq.size) expect(json_body[:items].any? { |fv| fv[:name] == "featureview2" && fv[:version] == 2}).to be true end @@ -671,6 +690,7 @@ expect_status_details(200) expect(json_body[:items].length).to eq(1) + expect(json_body[:count]).to eq(1) expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true expect(json_body[:items].all? { |fv| fv[:version] == 2 }).to be true end @@ -684,6 +704,7 @@ expect_status_details(200) expect(json_body[:items].length).to eq(2) + expect(json_body[:count]).to eq(3) end it "should be able to get a list of feature view offset" do @@ -700,6 +721,7 @@ ids_with_offset = json_body[:items].map { |o| o[:id] } expect(json_body[:items].length).to eq(2) + expect(json_body[:count]).to eq(3) for i in 0..json_body[:items].length-1 do expect(ids[i+1]).to eq(ids_with_offset[i]) end diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb index 6494d3f626..a94b3298b8 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb @@ -40,7 +40,9 @@ def create_test_files def get_storage_connectors(project_id, featurestore_id, type) json_result = get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/" connectors = JSON.parse(json_result) - connectors.select{|c| c['storageConnectorType'].eql?(type)}.map { |c| c.with_indifferent_access } + unless connectors["items"].nil? + connectors["items"].select{|c| c['storageConnectorType'].eql?(type)}.map { |c| c.with_indifferent_access } + end end def get_storage_connector(project_id, featurestore_id, name) diff --git a/hopsworks-IT/src/test/ruby/spec/model_spec.rb b/hopsworks-IT/src/test/ruby/spec/model_spec.rb index ad85e5ee40..3d38b655c8 100644 --- a/hopsworks-IT/src/test/ruby/spec/model_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/model_spec.rb @@ -235,6 +235,16 @@ expect(json_body[:items].count).to eq 3 json_body[:items].each {|model| expect(model[:framework]).to eq "TENSORFLOW"} end + it "should get 4 models with version 1" do + get_models(@project[:id], "?filter_by=version:1") + expect_status_details(200) + expect(json_body[:items].count).to eq 4 + end + it "should get 4 models with latest_version" do + get_models(@project[:id], "?filter_by=latest_version") + expect_status_details(200) + expect(json_body[:items].count).to eq 4 + end it "should get 1 Python model with name_eq model_python" do get_models(@project[:id], "?filter_by=name_eq:model_python") expect_status_details(200) diff --git a/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb b/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb index 4e5d75869a..8bf846c9fb 100644 --- a/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb @@ -588,7 +588,7 @@ json_result, connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") expect_status_details(201) - json_result, connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") + json_result, @connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") expect_status_details(201) end @@ -600,7 +600,8 @@ get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors" expect_status_details(200) - expect(json_body.length).to eq(5) + expect(json_body[:items].size).to eq(4) + expect(json_body[:count]).to eq(4) end it "should be able to get a list of connectors in the featurestore sorted by id" do @@ -610,10 +611,11 @@ # Get the list get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc" expect_status_details(200) - ids = json_body.map { |o| o[:id] } + ids = json_body[:items].map { |o| o[:id] } sorted_ids = ids.sort - expect(json_body.length).to eq(5) + expect(json_body[:items].size).to eq(4) + expect(json_body[:count]).to eq(4) expect(ids).to eq(sorted_ids) end @@ -624,10 +626,11 @@ # Get the list get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=NAME:asc" expect_status_details(200) - names = json_body.map { |o| "#{o[:name]}" } + names = json_body[:items].map { |o| "#{o[:name]}" } sorted_names = names.sort_by(&:downcase) - expect(json_body.length).to eq(5) + expect(json_body[:items].size).to eq(4) + expect(json_body[:count]).to eq(4) expect(names).to eq(sorted_names) end @@ -639,8 +642,35 @@ get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?filter_by=TYPE:HOPSFS" expect_status_details(200) - expect(json_body.length).to eq(4) - expect(json_body.all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true + expect(json_body[:items].size).to eq(4) + expect(json_body[:count]).to eq(4) + expect(json_body[:items].all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true + end + + it "should be able to get a list of connectors in the featurestore filtered by name" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + + # Get the list + get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?filter_by=NAME:" + @connector_name + expect_status_details(200) + + expect(json_body[:items].size).to eq(1) + expect(json_body[:count]).to eq(1) + expect(json_body[:items].all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true + end + + it "should be able to get a list of connectors in the featurestore filtered by name like" do + project = get_project + featurestore_id = get_featurestore_id(project.id) + + # Get the list + get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?filter_by=NAME_LIKE:hopsfs_connector_" + expect_status_details(200) + + expect(json_body[:items].size).to eq(3) + expect(json_body[:count]).to eq(3) + expect(json_body[:items].all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true end it "should be able to get a list of connectors in the featurestore limit" do @@ -651,7 +681,8 @@ get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc&limit=2" expect_status_details(200) - expect(json_body.length == 2).to be true + expect(json_body[:items].size == 2).to be true + expect(json_body[:count]).to eq(4) end it "should be able to get a list of connectors in the featurestore offset" do @@ -661,14 +692,15 @@ # Get the list get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc" expect_status_details(200) - ids = json_body.map { |o| o[:id] } + ids = json_body[:items].map { |o| o[:id] } get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc&offset=1" expect_status_details(200) - ids_with_offset = json_body.map { |o| o[:id] } + ids_with_offset = json_body[:items].map { |o| o[:id] } - expect(json_body.length).to eq(4) - for i in 0..json_body.length-1 do + expect(json_body[:items].size).to eq(3) + expect(json_body[:count]).to eq(4) + for i in 0..json_body[:items].size-1 do expect(ids[i+1]).to eq(ids_with_offset[i]) end end diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupBuilder.java index d170ded6ab..253f79d83c 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupBuilder.java @@ -16,21 +16,22 @@ package io.hops.hopsworks.api.featurestore.featuregroup; -import com.google.common.collect.Lists; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ServiceException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import java.util.List; +import javax.ws.rs.core.UriInfo; @Stateless @TransactionAttribute(TransactionAttributeType.NEVER) @@ -51,13 +52,16 @@ public FeaturegroupDTO build(Featuregroup featuregroup, Project project, Users u includeExpectationSuite); } - public List build(List featuregroups, Project project, Users user, - ResourceRequest resourceRequest) throws ServiceException, FeaturestoreException { - List featuregroupDTOS = Lists.newArrayList(); - for (Featuregroup featuregroup : featuregroups) { - featuregroupDTOS.add(build(featuregroup, project, user, resourceRequest)); + public FeaturegroupDTO build(AbstractFacade.CollectionInfo featuregroups, Featurestore featurestore, + Project project, Users user, ResourceRequest resourceRequest, UriInfo uriInfo) + throws ServiceException, FeaturestoreException { + FeaturegroupDTO featuregroupDTO = new FeaturegroupDTO(); + featuregroupDTO.setHref(uriInfo.getRequestUri()); + for (Featuregroup featuregroup : featuregroups.getItems()) { + featuregroupDTO.addItem(build(featuregroup, project, user, resourceRequest)); } - return featuregroupDTOS; + featuregroupDTO.setCount(featuregroups.getCount()); + return featuregroupDTO; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java index a89b7b80c9..10eafc8dfd 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java @@ -67,6 +67,7 @@ import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -98,7 +99,6 @@ import java.io.IOException; import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -217,18 +217,21 @@ public Response getFeaturegroupsForFeaturestore( @Context HttpServletRequest req, @Context - SecurityContext sc) + SecurityContext sc, + @Context + UriInfo uriInfo) throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); Project project = getProject(); Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(featureGroupBeanParam); - List featuregroups = featuregroupController - .getFeaturegroupsForFeaturestore(featurestore, project, user, convertToQueryParam(resourceRequest)); - GenericEntity> featuregroupsGeneric = - new GenericEntity>( - featuregroupBuilder.build(featuregroups, project, user, resourceRequest)) {}; - return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featuregroupsGeneric).build(); + + AbstractFacade.CollectionInfo featureGroups = featuregroupController.findByFeatureStore(featurestore, + resourceRequest.convertToQueryParam()); + FeaturegroupDTO featuregroupDTO = + featuregroupBuilder.build(featureGroups, featurestore, project, user, resourceRequest, uriInfo); + + return Response.ok().entity(featuregroupDTO).build(); } /** @@ -793,13 +796,4 @@ private ResourceRequest makeResourceRequest(FeatureGroupBeanParam param) { resourceRequest.setExpansions(param.getExpansion().getResources()); return resourceRequest; } - - private io.hops.hopsworks.common.dao.QueryParam convertToQueryParam(ResourceRequest resourceRequest) { - return new io.hops.hopsworks.common.dao.QueryParam( - resourceRequest.getOffset(), - resourceRequest.getLimit(), - resourceRequest.getFilter() == null ? new HashSet<>() : new HashSet<>(resourceRequest.getFilter()), - resourceRequest.getSort() == null ? new HashSet<>() : new HashSet<>(resourceRequest.getSort()) - ); - } -} +} \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewBuilder.java index c2831b04cb..9d1d802b4f 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewBuilder.java @@ -48,6 +48,7 @@ import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDatasetJoin; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; import javax.ejb.EJB; import javax.ejb.Stateless; @@ -123,14 +124,14 @@ private void setQuery(Project project, Users user, QueryDTO queryDTO, FeatureVie } } - public FeatureViewDTO build(List featureViews, ResourceRequest resourceRequest, Project project, - Users user, UriInfo uriInfo) + public FeatureViewDTO build(AbstractFacade.CollectionInfo featureviews, + Featurestore featurestore, Project project, Users user, UriInfo uriInfo, ResourceRequest resourceRequest) throws FeaturestoreException, ServiceException, MetadataException, DatasetException, FeatureStoreMetadataException { FeatureViewDTO featureViewDTO = new FeatureViewDTO(); featureViewDTO.setHref(uriInfo.getRequestUri()); - for (FeatureView featureView : featureViews) { + for (FeatureView featureView : featureviews.getItems()) { FeatureViewDTO featureViewItem = build(featureView, resourceRequest, project, user, uriInfo); featureViewItem.setHref(uriInfo.getRequestUriBuilder() .path("version") @@ -138,9 +139,7 @@ public FeatureViewDTO build(List featureViews, ResourceRequest reso .build()); featureViewDTO.addItem(featureViewItem); } - if (featureViews.size() > 1) { - featureViewDTO.setCount(Long.valueOf(featureViews.size())); - } + featureViewDTO.setCount(featureviews.getCount()); return featureViewDTO; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java index bf71b88dcc..ad7546cc41 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java @@ -24,7 +24,6 @@ import io.hops.hopsworks.api.filter.JWTNotRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.QueryParam; import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewDTO; @@ -44,6 +43,7 @@ import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -68,7 +68,6 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import java.io.IOException; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -163,12 +162,13 @@ public Response getAll( Project project = getProject(); Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(param); - List featureViews = featureViewController.getByFeatureStore(featurestore, - convertToQueryParam(resourceRequest)); - return Response.ok() - .entity(featureViewBuilder.build(featureViews, resourceRequest, project, user, uriInfo)) - .build(); + AbstractFacade.CollectionInfo featureviews = featureViewController.findByFeatureStore(featurestore, + resourceRequest.convertToQueryParam()); + FeatureViewDTO featureViewDTO = + featureViewBuilder.build(featureviews, featurestore, project, user, uriInfo, resourceRequest); + + return Response.ok().entity(featureViewDTO).build(); } @GET @@ -198,12 +198,13 @@ public Response getByName( Project project = getProject(); Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(param); - List featureViews = featureViewController.getByNameAndFeatureStore(name, featurestore, - convertToQueryParam(resourceRequest)); - return Response.ok() - .entity(featureViewBuilder.build(featureViews, resourceRequest, project, user, uriInfo)) - .build(); + AbstractFacade.CollectionInfo featureviews = featureViewController + .findByNameAndFeatureStore(name, featurestore, resourceRequest.convertToQueryParam()); + FeatureViewDTO featureViewDTO = + featureViewBuilder.build(featureviews, featurestore, project, user, uriInfo, resourceRequest); + + return Response.ok().entity(featureViewDTO).build(); } @GET @@ -413,13 +414,4 @@ private ResourceRequest makeResourceRequest(FeatureViewBeanParam param) { resourceRequest.setExpansions(param.getExpansion().getResources()); return resourceRequest; } - - private QueryParam convertToQueryParam(ResourceRequest resourceRequest) { - return new QueryParam( - resourceRequest.getOffset(), - resourceRequest.getLimit(), - resourceRequest.getFilter() == null ? new HashSet<>() : new HashSet<>(resourceRequest.getFilter()), - resourceRequest.getSort() == null ? new HashSet<>() : new HashSet<>(resourceRequest.getSort()) - ); - } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorBuilder.java new file mode 100644 index 0000000000..2a5f24b34c --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorBuilder.java @@ -0,0 +1,56 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ + +package io.hops.hopsworks.api.featurestore.storageconnector; + +import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; +import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; + +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; +import javax.ws.rs.core.UriInfo; + +@Stateless +@TransactionAttribute(TransactionAttributeType.NEVER) +public class FeaturestoreStorageConnectorBuilder { + + @EJB + private FeaturestoreStorageConnectorController storageConnectorController; + + public FeaturestoreStorageConnectorDTO build(AbstractFacade.CollectionInfo connectors, + Featurestore featurestore, Project project, Users user, ResourceRequest resourceRequest, UriInfo uriInfo) + throws FeaturestoreException { + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = new FeaturestoreStorageConnectorDTO(); + featurestoreStorageConnectorDTO.setHref(uriInfo.getRequestUri()); + + for (FeaturestoreConnector featurestoreConnector : connectors.getItems()) { + featurestoreStorageConnectorDTO.addItem( + storageConnectorController.convertToConnectorDTO(user, project, featurestoreConnector)); + } + featurestoreStorageConnectorDTO.setCount(connectors.getCount()); + return featurestoreStorageConnectorDTO; + } + +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java index 630c2416d9..63724ee55d 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java @@ -39,10 +39,12 @@ import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -70,8 +72,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; -import java.util.HashSet; -import java.util.List; +import javax.ws.rs.core.UriInfo; + import java.util.Set; import java.util.logging.Level; @@ -98,6 +100,8 @@ public class FeaturestoreStorageConnectorService extends FeaturestoreSubResource private StorageConnectorUtil storageConnectorUtil; @EJB private ConnectionChecker connectionChecker; + @EJB + private FeaturestoreStorageConnectorBuilder featurestoreStorageConnectorBuilder; @Inject private StorageConnectorProvenanceResource provenanceResource; @@ -128,19 +132,20 @@ protected FeaturestoreController getFeaturestoreController() { response = FeaturestoreStorageConnectorDTO.class, responseContainer = "List") public Response getStorageConnectors(@BeanParam StorageConnectorBeanParam storageConnectorBeanParam, @Context SecurityContext sc, - @Context HttpServletRequest req) + @Context HttpServletRequest req, + @Context UriInfo uriInfo) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); Project project = getProject(); Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(storageConnectorBeanParam); - List featurestoreStorageConnectorDTOS = storageConnectorController - .getConnectorsForFeaturestore(user, project, featurestore, convertToQueryParam(resourceRequest)); + AbstractFacade.CollectionInfo connectors = storageConnectorController + .findByFeaturestore(featurestore, resourceRequest.convertToQueryParam()); + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = + featurestoreStorageConnectorBuilder.build(connectors, featurestore, project, user, resourceRequest, uriInfo); - GenericEntity> featurestoreStorageConnectorsGeneric = - new GenericEntity>(featurestoreStorageConnectorDTOS) {}; - return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featurestoreStorageConnectorsGeneric) - .build(); + return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK) + .entity(featurestoreStorageConnectorDTO).build(); } @GET @@ -430,13 +435,4 @@ private ResourceRequest makeResourceRequest(StorageConnectorBeanParam param) { resourceRequest.setFilter(param.getFilters()); return resourceRequest; } - - private io.hops.hopsworks.common.dao.QueryParam convertToQueryParam(ResourceRequest resourceRequest) { - return new io.hops.hopsworks.common.dao.QueryParam( - resourceRequest.getOffset(), - resourceRequest.getLimit(), - resourceRequest.getFilter() == null ? new HashSet<>() : new HashSet<>(resourceRequest.getFilter()), - resourceRequest.getSort() == null ? new HashSet<>() : new HashSet<>(resourceRequest.getSort()) - ); - } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FilterBy.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FilterBy.java index 6f253bba35..bb11e9d947 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FilterBy.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FilterBy.java @@ -29,7 +29,7 @@ public FilterBy(String param) { this.filter = FeaturestoreConnectorFacade.Filters.valueOf(param.substring(0, param.indexOf(':')).toUpperCase()); this.param = param.substring(param.indexOf(':') + 1); } else { - this.filter = FeaturestoreConnectorFacade.Filters.valueOf(param); + this.filter = FeaturestoreConnectorFacade.Filters.valueOf(param.toUpperCase()); this.param = this.filter.getDefaultParam(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsBuilder.java index e0adbfaa2d..034bdea9ac 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsBuilder.java @@ -117,10 +117,7 @@ public JobDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project pr resourceRequest.getLimit(), resourceRequest.getFilter(), resourceRequest.getSort(), project); - //set the count - if (collectionInfo.getCount() > 0) { - dto.setCount(collectionInfo.getCount()); - } + dto.setCount(collectionInfo.getCount()); collectionInfo.getItems().forEach((job) -> dto.addItem(build(uriInfo, resourceRequest, (Jobs) job))); } return dto; diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/FilterBy.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/FilterBy.java index 1c64362b51..1eb9801089 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/FilterBy.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/FilterBy.java @@ -28,7 +28,7 @@ public FilterBy(String param) { this.filter = ModelVersionFacade.Filters.valueOf(param.substring(0, param.indexOf(':')).toUpperCase()); this.param = param.substring(param.indexOf(':') + 1); } else { - this.filter = ModelVersionFacade.Filters.valueOf(param); + this.filter = ModelVersionFacade.Filters.valueOf(param.toUpperCase()); this.param = this.filter.getDefaultParam(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java index 16307c98ea..9303a79c0e 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java @@ -22,7 +22,6 @@ import io.hops.hopsworks.api.tags.TagBuilder; import io.hops.hopsworks.api.user.UsersBuilder; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.AbstractFacade; import io.hops.hopsworks.common.dataset.FilePreviewMode; import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.dataset.util.DatasetPath; @@ -37,6 +36,7 @@ import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.persistence.entity.util.AbstractFacade; import io.hops.hopsworks.restutils.RESTCodes; import javax.ejb.EJB; @@ -115,10 +115,8 @@ public ModelDTO build(UriInfo uriInfo, dto.setCount(0l); if(dto.isExpand()) { try { - AbstractFacade.CollectionInfo models = modelVersionFacade.findByProject( - resourceRequest.getOffset(), resourceRequest.getLimit(), resourceRequest.getFilter(), - resourceRequest.getSort(), modelRegistryProject); - dto.setCount(models.getCount()); + AbstractFacade.CollectionInfo models = modelVersionFacade.findByProject(modelRegistryProject, + resourceRequest.convertToQueryParam()); String modelsDatasetPath = modelUtils.getModelsDatasetPath(userProject, modelRegistryProject); for(ModelVersion modelVersion: models.getItems()) { ModelDTO modelDTO = build(uriInfo, resourceRequest, user, userProject, modelRegistryProject, modelVersion, @@ -127,6 +125,7 @@ public ModelDTO build(UriInfo uriInfo, dto.addItem(modelDTO); } } + dto.setCount(Long.valueOf(models.getCount())); } catch(DatasetException e) { throw new ModelRegistryException(RESTCodes.ModelRegistryErrorCode.MODEL_LIST_FAILED, Level.FINE, "Unable to list models for project " + modelRegistryProject.getName(), e.getMessage(), e); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationBuilder.java index f79b29cb02..d1d9566009 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationBuilder.java @@ -84,10 +84,7 @@ public DefaultJobConfigurationDTO build(UriInfo uriInfo, ResourceRequest resourc resourceRequest.getLimit(), resourceRequest.getFilter(), resourceRequest.getSort(), project); - //set the count - if (collectionInfo.getCount() > 0) { - dto.setCount(collectionInfo.getCount()); - } + dto.setCount(collectionInfo.getCount()); collectionInfo.getItems().forEach((config) -> dto.addItem(build(uriInfo, resourceRequest, (DefaultJobConfiguration) config, ((DefaultJobConfiguration) config).getDefaultJobConfigurationPK().getType()))); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/api/ResourceRequest.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/api/ResourceRequest.java index 3bba41ab6a..a5692f183f 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/api/ResourceRequest.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/api/ResourceRequest.java @@ -17,8 +17,10 @@ import com.google.common.base.Strings; import io.hops.hopsworks.common.dao.AbstractFacade; +import io.hops.hopsworks.common.dao.QueryParam; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -148,6 +150,15 @@ public ResourceRequest get(Name name) { return null; } + public QueryParam convertToQueryParam() { + return new QueryParam( + getOffset(), + getLimit(), + getFilter() == null ? new HashSet<>() : new HashSet<>(getFilter()), + getSort() == null ? new HashSet<>() : new HashSet<>(getSort()) + ); + } + /** * Name of the resource requested by the user which needs to match the name of the resource in Hopsworks. */ diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSReindexer.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSReindexer.java index c1b0aff34a..6ec87743ab 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSReindexer.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSReindexer.java @@ -88,10 +88,10 @@ public void reindex() throws OpenSearchException, FeaturestoreException, Command } private void reindex(Featurestore featureStore) throws FeaturestoreException { - for (Featuregroup featuregroup : featureGroupFacade.findByFeaturestore(featureStore, null)) { + for (Featuregroup featuregroup : featureGroupFacade.getByFeatureStore(featureStore, null)) { reindex(featuregroup); } - for (FeatureView featureView : featureViewFacade.findByFeaturestore(featureStore, null)) { + for (FeatureView featureView : featureViewFacade.getByFeatureStore(featureStore, null)) { reindex(featureView); } for (TrainingDataset trainingDataset : trainingDatasetFacade.findByFeaturestore(featureStore)) { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java index 7310ee31cf..f888be2856 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java @@ -25,6 +25,7 @@ import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO; +import io.hops.hopsworks.common.featurestore.storageconnectors.StorageConnectorUtil; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetFacade; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.DistributedFsService; @@ -55,6 +56,7 @@ import java.util.Date; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -93,6 +95,8 @@ public class FeaturestoreController { private ProjectTeamFacade projectTeamFacade; @EJB private FeatureViewFacade featureViewFacade; + @EJB + private StorageConnectorUtil storageConnectorUtil; /* * Retrieves a list of all featurestores for a particular project @@ -320,10 +324,11 @@ public FeaturestoreDTO convertFeaturestoreToDTO(Featurestore featurestore) { } // add counters - featurestoreDTO.setNumFeatureGroups(featuregroupFacade.countByFeaturestore(featurestore)); + featurestoreDTO.setNumFeatureGroups(featuregroupFacade.countByFeatureStore(featurestore, null)); featurestoreDTO.setNumTrainingDatasets(trainingDatasetFacade.countByFeaturestore(featurestore)); - featurestoreDTO.setNumStorageConnectors(connectorFacade.countByFeaturestore(featurestore)); - featurestoreDTO.setNumFeatureViews(featureViewFacade.countByFeaturestore(featurestore)); + Set enabledScTypes = storageConnectorUtil.getEnabledStorageConnectorTypes(); + featurestoreDTO.setNumStorageConnectors(connectorFacade.countByFeaturestore(featurestore, enabledScTypes, null)); + featurestoreDTO.setNumFeatureViews(featureViewFacade.countByFeatureStore(featurestore, null)); return featurestoreDTO; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java index 514b77d2e4..6891f93f9f 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java @@ -20,6 +20,7 @@ import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; import io.hops.hopsworks.common.arrowflight.ArrowFlightController; import io.hops.hopsworks.common.commands.featurestore.search.SearchFSCommandLogger; +import io.hops.hopsworks.common.dao.AbstractFacade; import io.hops.hopsworks.common.dao.QueryParam; import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.activity.FeaturestoreActivityFacade; @@ -154,16 +155,14 @@ public class FeaturegroupController { protected ArrowFlightController arrowFlightController; /** - * Gets all featuregroups for a particular featurestore and project, using the userCerts to query Hive + * Gets all featuregroups for a particular featurestore * @param featurestore featurestore to query featuregroups for - * @param project - * @param user * @param queryParam - * @return list of feature groups + * @return CollectionInfo object */ - public List getFeaturegroupsForFeaturestore(Featurestore featurestore, Project project, Users user, - QueryParam queryParam) { - return featuregroupFacade.findByFeaturestore(featurestore, queryParam); + public AbstractFacade.CollectionInfo findByFeatureStore( + Featurestore featurestore, QueryParam queryParam) { + return featuregroupFacade.findByFeatureStore(featurestore, queryParam); } /** diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java index 9c622f4493..ec16176ca6 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupFacade.java @@ -155,6 +155,11 @@ public List findAll() { TypedQuery q = em.createNamedQuery("Featuregroup.findAll", Featuregroup.class); return q.getResultList(); } + + public CollectionInfo findByFeatureStore(Featurestore featurestore, QueryParam queryParam) { + return new CollectionInfo(countByFeatureStore(featurestore, queryParam), + getByFeatureStore(featurestore, queryParam)); + } /** * Retrieves all featuregroups for a particular featurestore @@ -163,7 +168,7 @@ public List findAll() { * @param queryParam * @return list of featuregroup entities */ - public List findByFeaturestore(Featurestore featurestore, QueryParam queryParam) { + public List getByFeatureStore(Featurestore featurestore, QueryParam queryParam) { String queryStr = buildQuery("SELECT fg FROM Featuregroup fg ", queryParam != null ? queryParam.getFilters(): null, queryParam != null ? queryParam.getSorts(): null, @@ -180,10 +185,28 @@ public List findByFeaturestore(Featurestore featurestore, QueryPar return query.getResultList(); } - public Long countByFeaturestore(Featurestore featurestore) { - return em.createNamedQuery("Featuregroup.countByFeaturestore", Long.class) - .setParameter("featurestore", featurestore) - .getSingleResult(); + /** + * Retrieves count of feature groups for a particular featurestore + * + * @param featurestore the featurestore to query + * @param queryParam + * @return number of feature group entities + */ + public Long countByFeatureStore(Featurestore featurestore, QueryParam queryParam) { + String queryStr = buildQuery("SELECT count(fg.id) FROM Featuregroup fg ", + queryParam != null ? queryParam.getFilters(): null, + null, + "fg.featurestore = :featurestore" + + " AND (fg.onDemandFeaturegroup IS NOT null" + + " OR fg.cachedFeaturegroup IS NOT null" + + " OR fg.streamFeatureGroup IS NOT null)"); + + TypedQuery query = em.createQuery(queryStr, Long.class) + .setParameter("featurestore", featurestore); + if (queryParam != null) { + setFilter(queryParam.getFilters(), query); + } + return query.getSingleResult(); } public List findByStorageConnectors(List storageConnectors) { @@ -253,6 +276,7 @@ private void setFilter(Set filters, Query q) private void setFilterQuery(FilterBy filter, Query q) { switch (Filters.valueOf(filter.getValue())) { case NAME: + case NAME_LIKE: q.setParameter(filter.getField(), filter.getParam()); break; case VERSION: @@ -265,6 +289,7 @@ private void setFilterQuery(FilterBy filter, Query q) { public enum Filters { NAME("NAME", "fg.name = :name", "name", ""), + NAME_LIKE("NAME_LIKE", "fg.name LIKE CONCAT('%', :name, '%') ", "name", ""), VERSION("VERSION", "fg.version = :version", "version", ""), LATEST_VERSION("LATEST_VERSION", String.format("%1$s.version = ( " + "SELECT MAX(%2$s.version) " + diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewController.java index 81fddefc77..3e2aacfd6e 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewController.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.hops.hopsworks.common.commands.featurestore.search.SearchFSCommandLogger; +import io.hops.hopsworks.common.dao.AbstractFacade; import io.hops.hopsworks.common.dao.QueryParam; import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; import io.hops.hopsworks.common.featurestore.activity.FeaturestoreActivityFacade; @@ -60,6 +61,7 @@ import javax.ejb.TransactionAttributeType; import javax.inject.Inject; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.List; @@ -120,9 +122,9 @@ public FeatureView createFeatureView(Project project, Users user, FeatureView fe } // Check that feature view doesn't already exists - if (!featureViewFacade - .findByNameVersionAndFeaturestore(featureView.getName(), featureView.getVersion(), featurestore) - .isEmpty()) { + if (featureViewFacade + .findByNameVersionAndFeaturestore(featureView.getName(), featureView.getVersion(), featurestore) + .isPresent()) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_VIEW_ALREADY_EXISTS, Level.FINE, "Feature view: " + featureView.getName() + ", version: " + featureView.getVersion()); } @@ -181,15 +183,20 @@ public List getAll() { return featureViewFacade.findAll(); } - public List getByFeatureStore(Featurestore featurestore, QueryParam queryParam) { - return featureViewFacade.findByFeaturestore(featurestore, queryParam); + public AbstractFacade.CollectionInfo findByFeatureStore( + Featurestore featurestore, QueryParam queryParam) { + return featureViewFacade.findByFeatureStore(featurestore, queryParam); } - public List getByNameAndFeatureStore(String name, Featurestore featurestore, QueryParam queryParam) + public AbstractFacade.CollectionInfo findByNameAndFeatureStore( + String name, Featurestore featurestore, QueryParam queryParam) throws FeaturestoreException { - List featureViews = featureViewFacade.findByNameAndFeaturestore( - name, featurestore, queryParam); - if (featureViews.isEmpty()) { + Set filters = queryParam.getFilters(); + // Use different parameter name so do not conflict with the default one. + filters.add(new FeatureViewFilterBy("name", name, "name1")); + AbstractFacade.CollectionInfo featureViews = + featureViewFacade.findByFeatureStore(featurestore, queryParam); + if (featureViews.getItems().isEmpty()) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_VIEW_NOT_FOUND, Level.FINE, String.format("There exists no feature view with the name %s.", name)); } @@ -207,24 +214,23 @@ public FeatureView getByIdAndFeatureStore(Integer id, Featurestore featureStore) public FeatureView getByNameVersionAndFeatureStore(String name, Integer version, Featurestore featurestore) throws FeaturestoreException { - List featureViews = featureViewFacade.findByNameVersionAndFeaturestore(name, version, featurestore); - if (featureViews.isEmpty()) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_VIEW_NOT_FOUND, - Level.FINE, String.format("There exists no feature view with the name %s and version %d.", name, version)); - } - return featureViews.get(0); + return featureViewFacade.findByNameVersionAndFeaturestore(name, version, featurestore) + .orElseThrow(() -> new FeaturestoreException( + RESTCodes.FeaturestoreErrorCode.FEATURE_VIEW_NOT_FOUND, + Level.FINE, + String.format("There exists no feature view with the name %s and version %d.", name, version))); } public void delete(Users user, Project project, Featurestore featurestore, String name) throws FeaturestoreException, JobException { - List featureViews = featureViewFacade.findByNameAndFeaturestore(name, featurestore); + List featureViews = findByNameAndFeatureStore(name, featurestore, null).getItems(); delete(user, project, featurestore, featureViews); } public void delete(Users user, Project project, Featurestore featurestore, String name, Integer version) throws FeaturestoreException, JobException { - List featureViews = featureViewFacade.findByNameVersionAndFeaturestore(name, version, featurestore); - delete(user, project, featurestore, featureViews); + FeatureView featureView = getByNameVersionAndFeatureStore(name, version, featurestore); + delete(user, project, featurestore, Arrays.asList(featureView)); } private void delete(Users user, Project project, Featurestore featurestore, List featureViews) diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java index 2000b3d6a0..58f7b251c5 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featureview/FeatureViewFacade.java @@ -49,7 +49,19 @@ public List findAll() { return q.getResultList(); } - public List findByFeaturestore(Featurestore featurestore, QueryParam queryParam) { + public CollectionInfo findByFeatureStore(Featurestore featurestore, QueryParam queryParam) { + return new CollectionInfo(countByFeatureStore(featurestore, queryParam), + getByFeatureStore(featurestore, queryParam)); + } + + /** + * Retrieves feature views for a particular featurestore + * + * @param featurestore the featurestore to query + * @param queryParam + * @return feature view entities + */ + public List getByFeatureStore(Featurestore featurestore, QueryParam queryParam) { Map extraParam = new HashMap<>(); extraParam.put("featurestore", featurestore); String queryStr = buildQuery("SELECT fv FROM FeatureView fv ", @@ -60,10 +72,25 @@ public List findByFeaturestore(Featurestore featurestore, QueryPara return q.getResultList(); } - public Long countByFeaturestore(Featurestore featurestore) { - return em.createNamedQuery("FeatureView.countByFeaturestore", Long.class) - .setParameter("featurestore", featurestore) - .getSingleResult(); + /** + * Retrieves count of feature view for a particular featurestore + * + * @param featurestore the featurestore to query + * @param queryParam + * @return number of feature view entities + */ + public Long countByFeatureStore(Featurestore featurestore, QueryParam queryParam) { + String queryStr = buildQuery("SELECT count(fv.id) FROM FeatureView fv ", + queryParam != null ? queryParam.getFilters(): null, + null, + "fv.featurestore = :featurestore "); + + TypedQuery query = em.createQuery(queryStr, Long.class) + .setParameter("featurestore", featurestore); + if (queryParam != null) { + setFilter(queryParam.getFilters(), query); + } + return query.getSingleResult(); } public Optional findByIdAndFeatureStore(Integer id, Featurestore featureStore) { @@ -77,45 +104,31 @@ public Optional findByIdAndFeatureStore(Integer id, Featurestore fe return Optional.empty(); } } - - public List findByNameAndFeaturestore(String name, Featurestore featurestore) { - QueryParam queryParam = new QueryParam(); - return findByNameAndFeaturestore(name, featurestore, queryParam); - } - public List findByNameAndFeaturestore(String name, Featurestore featurestore, - QueryParam queryParam) { - Set filters = queryParam.getFilters(); - // Use different parameter name so do not conflict with the default one. - filters.add(new FeatureViewFilterBy("name", name, "name1")); - return findByFeaturestore(featurestore, queryParam); - } - - public List findByNameVersionAndFeaturestore(String name, Integer version, Featurestore featurestore) { - QueryParam queryParam = new QueryParam(); - return findByNameVersionAndFeaturestore(name, version, featurestore, queryParam); - } - - public List findByNameVersionAndFeaturestore(String name, Integer version, - Featurestore featurestore, QueryParam queryParam) { - Set filters = queryParam.getFilters(); - // Use different parameter name so do not conflict with the default one. - filters.add(new FeatureViewFilterBy("name", name, "name1")); - filters.add(new FeatureViewFilterBy("version", version.toString(), "version1")); - return findByFeaturestore(featurestore, queryParam); + public Optional findByNameVersionAndFeaturestore( + String name, Integer version, Featurestore featureStore) { + try { + return Optional.of( + em.createNamedQuery("FeatureView.findByNameVersionAndFeaturestore", FeatureView.class) + .setParameter("name", name) + .setParameter("version", version) + .setParameter("featurestore", featureStore) + .getSingleResult()); + } catch (NoResultException e) { + return Optional.empty(); + } } public Integer findLatestVersion(String name, Featurestore featurestore) { - TypedQuery query = em.createNamedQuery("FeatureView.findByFeaturestoreAndNameOrderedByDescVersion", - FeatureView.class); + TypedQuery query = em.createNamedQuery("FeatureView.findMaxVersionByFeaturestoreAndName", Integer.class); query.setParameter("name", name); query.setParameter("featurestore", featurestore); - List results = query.getResultList(); - if (results != null && !results.isEmpty()) { - return results.get(0).getVersion(); - } else { - return null; - } + return query.getSingleResult(); + } + + public List findByFeatureGroup(Integer featureGroupId) { + return em.createNamedQuery("FeatureView.findByFeatureGroup", FeatureView.class) + .setParameter("featureGroupId", featureGroupId).getResultList(); } private Query makeQuery(String queryStr, QueryParam queryParam, Map extraParam) { @@ -144,6 +157,7 @@ private void setFilter(Set filter, Query q) { private void setFilterQuery(AbstractFacade.FilterBy filterBy, Query q) { switch (FeatureViewFacade.Filters.valueOf(filterBy.toString())) { case NAME: + case NAME_LIKE: q.setParameter(filterBy.getParam(), filterBy.getValue()); break; case VERSION: @@ -154,11 +168,6 @@ private void setFilterQuery(AbstractFacade.FilterBy filterBy, Query q) { } } - public List findByFeatureGroup(Integer featureGroupId) { - return em.createNamedQuery("FeatureView.findByFeatureGroup", FeatureView.class) - .setParameter("featureGroupId", featureGroupId).getResultList(); - } - @Override protected EntityManager getEntityManager() { return em; @@ -166,6 +175,8 @@ protected EntityManager getEntityManager() { public enum Filters { NAME("NAME", String.format("%s.name = :%%s ", FeatureView.TABLE_NAME_ALIAS), "name", ""), + NAME_LIKE("NAME_LIKE", String.format("%s.name LIKE CONCAT('%%%%', :%%s, '%%%%') ", FeatureView.TABLE_NAME_ALIAS), + "name", ""), VERSION("VERSION", String.format("%s.version = :%%s ", FeatureView.TABLE_NAME_ALIAS), "version", ""), LATEST_VERSION("LATEST_VERSION", String.format("%1$s.version = ( " + "SELECT MAX(%2$s.version) " + diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java index aa2b13c208..8f5e2f94de 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java @@ -29,6 +29,7 @@ import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.persistence.Query; +import javax.persistence.TypedQuery; import java.util.List; import java.util.Optional; import java.util.Set; @@ -76,17 +77,6 @@ public List findByFeaturestore(Featurestore featurestore) .getResultList(); } - public Optional findByFeaturestoreId(Featurestore featurestore, Integer id) { - try { - return Optional.of(em.createNamedQuery("FeaturestoreConnector.findByFeaturestoreId", FeaturestoreConnector.class) - .setParameter("featurestore", featurestore) - .setParameter("id", id) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - public Optional findByFeaturestoreName(Featurestore featurestore, String name) { try { return Optional.of(em.createNamedQuery("FeaturestoreConnector.findByFeaturestoreName", @@ -99,7 +89,13 @@ public Optional findByFeaturestoreName(Featurestore featu } } - public List findByType(Featurestore featurestore, Set types, + public CollectionInfo findByFeaturestoreAndTypes(Featurestore featurestore, Set types, + QueryParam queryParam) { + return new CollectionInfo(countByFeaturestore(featurestore, types, queryParam), + getByType(featurestore, types, queryParam)); + } + + private List getByType(Featurestore featurestore, Set types, QueryParam queryParam) { String queryStr = buildQuery("SELECT fsConn FROM FeaturestoreConnector fsConn ", queryParam != null ? queryParam.getFilters(): null, @@ -120,10 +116,27 @@ public void deleteByFeaturestoreName(Featurestore featurestore, String name) { findByFeaturestoreName(featurestore, name).ifPresent(this::remove); } - public Long countByFeaturestore(Featurestore featurestore) { - return em.createNamedQuery("FeaturestoreConnector.countByFeaturestore", Long.class) + /** + * Retrieves count of storage connectors for a particular featurestore + * + * @param featurestore the featurestore to query + * @param queryParam + * @return number of storage connector entities + */ + public Long countByFeaturestore(Featurestore featurestore, Set types, + QueryParam queryParam) { + String queryStr = buildQuery("SELECT count(fsConn.id) FROM FeaturestoreConnector fsConn ", + queryParam != null ? queryParam.getFilters(): null, + null, + "fsConn.featurestore = :featurestore AND fsConn.connectorType IN :types"); + + TypedQuery query = em.createQuery(queryStr, Long.class) .setParameter("featurestore", featurestore) - .getSingleResult(); + .setParameter("types", types); + if (queryParam != null) { + setFilter(queryParam.getFilters(), query); + } + return query.getSingleResult(); } private void setFilter(Set filters, Query q) { @@ -137,6 +150,10 @@ private void setFilter(Set filters, Query q) private void setFilterQuery(FilterBy filter, Query q) { switch (Filters.valueOf(filter.getValue())) { + case NAME: + case NAME_LIKE: + q.setParameter(filter.getField(), filter.getParam()); + break; case TYPE: q.setParameter(filter.getField(), FeaturestoreConnectorType.valueOf(filter.getParam().toUpperCase())); break; @@ -146,6 +163,8 @@ private void setFilterQuery(FilterBy filter, Query q) { } public enum Filters { + NAME("NAME", "fsConn.name = :name", "name", ""), + NAME_LIKE("NAME_LIKE", "fsConn.name LIKE CONCAT('%', :name, '%') ", "name", ""), TYPE("TYPE", "fsConn.connectorType = :type", "type", ""); private final String value; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java index c29c517ae7..a7466e6934 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java @@ -17,6 +17,7 @@ package io.hops.hopsworks.common.featurestore.storageconnectors; import com.google.common.base.Strings; +import io.hops.hopsworks.common.dao.AbstractFacade; import io.hops.hopsworks.common.dao.QueryParam; import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; @@ -60,8 +61,6 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.transaction.Transactional; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -110,21 +109,17 @@ public class FeaturestoreStorageConnectorController { private static final String KAFKA_STORAGE_CONNECTOR_NAME = "kafka_connector"; /** - * Returns a list with DTOs of all storage connectors for a featurestore + * Returns a list of all storage connectors for a featurestore * - * @param user the user making the request - * @param project the project to query * @param featurestore the featurestore to query * @param queryParam the queryParam - * @return List of JSON/XML DTOs of the storage connectors + * @return CollectionInfo object */ - public List getConnectorsForFeaturestore(Users user, Project project, + public AbstractFacade.CollectionInfo findByFeaturestore( Featurestore featurestore, QueryParam queryParam) throws FeaturestoreException { Set enabledScTypes = storageConnectorUtil.getEnabledStorageConnectorTypes(); - List connectors = featurestoreConnectorFacade.findByType(featurestore, - enabledScTypes, queryParam); - return convertToConnectorDTOs(user, project, connectors); + return featurestoreConnectorFacade.findByFeaturestoreAndTypes(featurestore, enabledScTypes, queryParam); } public FeaturestoreConnector getConnectorWithName(Users user, Project project, @@ -153,18 +148,6 @@ public FeaturestoreStorageConnectorDTO getConnectorDTOWithName(Users user, Proje return convertToConnectorDTO(user, project, featurestoreConnector); } - public List convertToConnectorDTOs(Users user, Project project, - List featurestoreConnectors) - throws FeaturestoreException { - List featurestoreStorageConnectorDTOS = new ArrayList<>(); - - for (FeaturestoreConnector featurestoreConnector : featurestoreConnectors) { - featurestoreStorageConnectorDTOS.add(convertToConnectorDTO(user, project, featurestoreConnector)); - } - - return featurestoreStorageConnectorDTOS; - } - public FeatureStoreKafkaConnectorDTO getKafkaConnector(Project project) throws FeaturestoreException { Featurestore featureStore = featurestoreController.getProjectFeaturestore(project); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/models/version/ModelVersionFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/models/version/ModelVersionFacade.java index 7933850d55..66639c6d06 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/models/version/ModelVersionFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/models/version/ModelVersionFacade.java @@ -17,6 +17,7 @@ package io.hops.hopsworks.common.models.version; import io.hops.hopsworks.common.dao.AbstractFacade; +import io.hops.hopsworks.common.dao.QueryParam; import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; import io.hops.hopsworks.persistence.entity.project.Project; @@ -26,6 +27,7 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; import javax.persistence.TypedQuery; +import java.util.List; import java.util.Set; /** @@ -46,26 +48,39 @@ protected EntityManager getEntityManager() { return em; } - public CollectionInfo findByProject(Integer offset, Integer limit, - Set filters, - Set sorts, - Project project) { - - String queryStr = buildQuery( - "SELECT * FROM hopsworks.`model_version` JOIN `hopsworks`.model ON `hopsworks`.model_version.model_id=model.id ", - filters, sorts, "`hopsworks`.model.project_id = ?project_id "); - - String queryCountStr = - buildQuery("SELECT COUNT(DISTINCT concat(model_version.model_id, model_version.version)) " + - "FROM hopsworks.`model_version` JOIN `hopsworks`.model ON `hopsworks`.model_version.model_id=model.id ", - filters, sorts, "`hopsworks`.model.project_id = ?project_id "); - - Query query = em.createNativeQuery(queryStr, ModelVersion.class).setParameter("project_id", project.getId()); - Query queryCount = em.createNativeQuery(queryCountStr).setParameter("project_id", project.getId()); - setFilter(filters, query); - setFilter(filters, queryCount); - setOffsetAndLim(offset, limit, query); - return new CollectionInfo((Long)queryCount.getSingleResult(), query.getResultList()); + public CollectionInfo findByProject(Project project, QueryParam queryParam) { + return new CollectionInfo(countByProject(project, queryParam), getByProject(project, queryParam)); + } + + private List getByProject(Project project, QueryParam queryParam) { + String queryStr = buildQuery("SELECT * FROM hopsworks.`model_version` mv " + + "JOIN `hopsworks`.model m ON mv.model_id=m.id ", + queryParam != null ? queryParam.getFilters(): null, + queryParam != null ? queryParam.getSorts(): null, + "m.project_id = ?project_id "); + + Query query = em.createNativeQuery(queryStr, ModelVersion.class) + .setParameter("project_id", project.getId()); + if (queryParam != null) { + setFilter(queryParam.getFilters(), query); + setOffsetAndLim(queryParam.getOffset(), queryParam.getLimit(), query); + } + return query.getResultList(); + } + + private Long countByProject(Project project, QueryParam queryParam) { + String queryStr = buildQuery("SELECT COUNT(mv.id) FROM hopsworks.`model_version` mv " + + "JOIN `hopsworks`.model m ON mv.model_id=m.id ", + queryParam != null ? queryParam.getFilters(): null, + null, + "m.project_id = ?project_id "); + + Query query = em.createNativeQuery(queryStr) + .setParameter("project_id", project.getId()); + if (queryParam != null) { + setFilter(queryParam.getFilters(), query); + } + return (Long) query.getSingleResult(); } public ModelVersion findByProjectAndMlId(Integer modelId, Integer version) { @@ -91,18 +106,20 @@ private void setFilterQuery(AbstractFacade.FilterBy filterBy, Query q) { switch (Filters.valueOf(filterBy.getValue())) { case NAME_EQ: case NAME_LIKE: - case VERSION: q.setParameter(filterBy.getField(), filterBy.getParam()); break; + case VERSION: + q.setParameter(filterBy.getField(), Integer.valueOf(filterBy.getParam())); + break; default: break; } } public enum Sorts { - NAME("NAME", "`hopsworks`.model.name " , "ASC"), - METRIC("METRIC", "JSON_VALUE(`metrics`, '$.attributes.METRIC') IS NULL, " + - "CAST(JSON_VALUE(`metrics`, '$.attributes.METRIC') AS FLOAT) ", + NAME("NAME", "m.name " , "ASC"), + METRIC("METRIC", "JSON_VALUE(mv.metrics, '$.attributes.METRIC') IS NULL, " + + "CAST(JSON_VALUE(mv.metrics, '$.attributes.METRIC') AS FLOAT) ", "ASC"); //sort twice needed to make sure nulls always at the end of sorted items private final String value; private final String sql; @@ -151,14 +168,19 @@ public void setJsonSortKey(String jsonSortKey) { } public enum Filters { NAME_EQ ("NAME_EQ", - "`hopsworks`.model.name = ?name", + "m.name = ?name", "name", ""), NAME_LIKE ("NAME_LIKE", - "`hopsworks`.model.name LIKE CONCAT('%', ?name, '%') ", + "m.name LIKE CONCAT('%', ?name, '%') ", "name", " "), VERSION ("VERSION", - "`hopsworks`.model_version.version = ?version ", - "version", ""); + "mv.version = ?version ", + "version", ""), + LATEST_VERSION("LATEST_VERSION", String.format("%1$s.version = ( " + + "SELECT MAX(%2$s.version) " + + "FROM `hopsworks`.model_version %2$s " + + "WHERE %1$s.model_id = %2$s.model_id " + + ") ", "mv", "mv2"), null, null); private final String value; private final String sql; private final String field; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/QuotasEnforcement.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/QuotasEnforcement.java index 1dd7027367..f9c6ce2ddb 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/QuotasEnforcement.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/QuotasEnforcement.java @@ -89,7 +89,7 @@ public void enforceFeaturegroupsQuota(Featurestore featurestore, boolean onlineE } private List getFeaturegroups(Featurestore featurestore) { - return featuregroupFacade.findByFeaturestore(featurestore, null); + return featuregroupFacade.getByFeatureStore(featurestore, null); } public void enforceTrainingDatasetsQuota(Featurestore featurestore) throws QuotaEnforcementException { diff --git a/hopsworks-common/src/test/io/hops/hopsworks/common/security/TestQuotasEnforcement.java b/hopsworks-common/src/test/io/hops/hopsworks/common/security/TestQuotasEnforcement.java index c9dc60f52d..f8bca4a394 100644 --- a/hopsworks-common/src/test/io/hops/hopsworks/common/security/TestQuotasEnforcement.java +++ b/hopsworks-common/src/test/io/hops/hopsworks/common/security/TestQuotasEnforcement.java @@ -67,7 +67,7 @@ public void testQuotasOnlineEnabledFeaturegroups() throws Exception { fg2.setOnlineEnabled(true); mockedOnlineEnabledFeaturegroups.add(fg2); - Mockito.when(featuregroupFacade.findByFeaturestore(Mockito.any(), Mockito.any())).thenReturn(mockedOnlineEnabledFeaturegroups); + Mockito.when(featuregroupFacade.getByFeatureStore(Mockito.any(), Mockito.any())).thenReturn(mockedOnlineEnabledFeaturegroups); Settings settings = Mockito.mock(Settings.class); @@ -114,7 +114,7 @@ public void testQuotasOnlineDisabledFeaturegroups() throws Exception { fg2.setOnlineEnabled(false); mockedOnlineDisabledFeaturegroups.add(fg2); - Mockito.when(featuregroupFacade.findByFeaturestore(Mockito.any(), Mockito.any())).thenReturn(mockedOnlineDisabledFeaturegroups); + Mockito.when(featuregroupFacade.getByFeatureStore(Mockito.any(), Mockito.any())).thenReturn(mockedOnlineDisabledFeaturegroups); Settings settings = Mockito.mock(Settings.class); @@ -160,7 +160,7 @@ public void testIgnoreFeaturegroupQuotas() throws Exception { Mockito.when(settings.getQuotasOnlineDisabledFeaturegroups()).thenReturn(-1L); qe.enforceFeaturegroupsQuota(fs, false); qe.enforceFeaturegroupsQuota(fs, true); - Mockito.verify(featuregroupFacade, Mockito.never()).findByFeaturestore(Mockito.any(), Mockito.any()); + Mockito.verify(featuregroupFacade, Mockito.never()).findByFeatureStore(Mockito.any(), Mockito.any()); } @Test diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/Featuregroup.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/Featuregroup.java index d2d4f393e4..d166719306 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/Featuregroup.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/Featuregroup.java @@ -69,16 +69,6 @@ + " AND (fg.onDemandFeaturegroup IS NOT null " + "OR fg.cachedFeaturegroup IS NOT null " + "OR fg.streamFeatureGroup IS NOT null)"), - @NamedQuery(name = "Featuregroup.findByFeaturestore", query = "SELECT fg FROM Featuregroup fg " + - "WHERE fg.featurestore = :featurestore" - + " AND (fg.onDemandFeaturegroup IS NOT null " - + "OR fg.cachedFeaturegroup IS NOT null " - + "OR fg.streamFeatureGroup IS NOT null)"), - @NamedQuery(name = "Featuregroup.countByFeaturestore", query = "SELECT count(fg.id) FROM Featuregroup fg " + - "WHERE fg.featurestore = :featurestore" - + " AND (fg.onDemandFeaturegroup IS NOT null " - + "OR fg.cachedFeaturegroup IS NOT null " - + "OR fg.streamFeatureGroup IS NOT null)"), @NamedQuery(name = "Featuregroup.findByFeaturestoreAndId", query = "SELECT fg FROM Featuregroup fg " + "WHERE fg.featurestore = :featurestore AND fg.id = :id"), @NamedQuery(name = "Featuregroup.findByFeaturestoreAndNameVersion", query = "SELECT fg FROM Featuregroup fg " + diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featureview/FeatureView.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featureview/FeatureView.java index b101ea0b33..7a8f29f8cd 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featureview/FeatureView.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featureview/FeatureView.java @@ -51,18 +51,13 @@ @XmlRootElement @NamedQueries({ @NamedQuery(name = "FeatureView.findAll", query = "SELECT fv FROM FeatureView fv"), - @NamedQuery(name = "FeatureView.findById", query = "SELECT fv FROM FeatureView fv WHERE fv.id = :id"), @NamedQuery(name = "FeatureView.findByIdAndFeaturestore", query = "SELECT fv FROM FeatureView fv " + "WHERE fv.featurestore = :featurestore AND fv.id = :id"), - @NamedQuery(name = "FeatureView.findByFeaturestore", query = "SELECT fv FROM FeatureView fv " + - "WHERE fv.featurestore = :featurestore"), - @NamedQuery(name = "FeatureView.findByFeaturestoreAndNameVersion", - query = "SELECT fv FROM FeatureView fv WHERE fv.featurestore = :featurestore " + - "AND fv.name= :name AND fv.version = :version"), - @NamedQuery(name = "FeatureView.findByFeaturestoreAndNameOrderedByDescVersion", query = "SELECT fv FROM " + - "FeatureView fv WHERE fv.featurestore = :featurestore AND fv.name = :name ORDER BY fv.version DESC"), - @NamedQuery(name = "FeatureView.countByFeaturestore", query = "SELECT count(fv.id) FROM FeatureView fv " + - "WHERE fv.featurestore = :featurestore"), + @NamedQuery(name = "FeatureView.findByNameVersionAndFeaturestore", query = "SELECT fv FROM FeatureView fv " + + "WHERE fv.featurestore = :featurestore AND fv.name = :name AND fv.version = :version"), + @NamedQuery(name = "FeatureView.findMaxVersionByFeaturestoreAndName", query = + "SELECT MAX(fv.version) FROM FeatureView fv " + + "WHERE fv.featurestore = :featurestore AND fv.name = :name"), @NamedQuery(name = "FeatureView.findByFeatureGroup", query = "SELECT DISTINCT fv FROM FeatureView fv " + "JOIN fv.features tdf WHERE tdf.featureGroup.id = :featureGroupId")}) public class FeatureView implements Serializable { diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java index 7be587b4cc..f3ca3d8442 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java @@ -52,8 +52,6 @@ @Table(name = "feature_store_connector", catalog = "hopsworks") @XmlRootElement @NamedQueries({ - @NamedQuery(name = "FeaturestoreConnector.findAll", - query = "SELECT fsConn FROM FeaturestoreConnector fsConn"), @NamedQuery(name = "FeaturestoreConnector.findById", query = "SELECT fsConn FROM FeaturestoreConnector fsConn WHERE fsConn.id = :id"), @NamedQuery(name = "FeaturestoreConnector.findByIdType", @@ -61,17 +59,9 @@ "WHERE fsConn.id = :id AND fsConn.connectorType= :type"), @NamedQuery(name = "FeaturestoreConnector.findByFeaturestore", query = "SELECT fsConn FROM FeaturestoreConnector fsConn WHERE fsConn.featurestore = :featurestore"), - @NamedQuery(name = "FeaturestoreConnector.countByFeaturestore", - query = "SELECT count(fsConn.id) FROM FeaturestoreConnector fsConn WHERE fsConn.featurestore = :featurestore"), - @NamedQuery(name = "FeaturestoreConnector.findByFeaturestoreId", - query = "SELECT fsConn FROM FeaturestoreConnector fsConn " + - "WHERE fsConn.featurestore = :featurestore AND fsConn.id = :id"), @NamedQuery(name = "FeaturestoreConnector.findByFeaturestoreName", query = "SELECT fsConn FROM FeaturestoreConnector fsConn " + - "WHERE fsConn.featurestore = :featurestore AND fsConn.name = :name"), - @NamedQuery(name = "FeaturestoreConnector.findByType", - query = "SELECT fsConn FROM FeaturestoreConnector fsConn " + - "WHERE fsConn.featurestore = :featurestore AND fsConn.connectorType IN :types")}) + "WHERE fsConn.featurestore = :featurestore AND fsConn.name = :name")}) public class FeaturestoreConnector implements Serializable { private static final long serialVersionUID = 1L; @Id