[FSTORE-1400] Make sure we return the totalItemsCount without applyin…

…g the pagination (#1798)

hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

@@ -3180,19 +3180,19 @@
         featurestore_id = get_featurestore_id(project.id)
 
         # create some feature groups
-        create_cached_featuregroup(project.id, featurestore_id)
+        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:1)
         expect_status_details(201)
 
-        create_cached_featuregroup(project.id, featurestore_id)
+        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:2)
         expect_status_details(201)
 
-        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:1)
+        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:3)
         expect_status_details(201)
 
-        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:2)
+        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup2", version:1)
         expect_status_details(201)
 
-        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup1", version:3)
+        create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup3", version:1)
         expect_status_details(201)
       end
 
@@ -3207,25 +3207,25 @@
         get get_featuregroups_endpoint
         parsed_json = JSON.parse(response.body)
         expect_status_details(200)
-        old_count = parsed_json.length
+        old_count = parsed_json["items"].size
 
         # create fg
-        json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id)
+        json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id, featuregroup_name:"featuregroup4", version:1)
         expect_status_details(201)
 
         # get fgs
         get get_featuregroups_endpoint
         parsed_json = JSON.parse(response.body)
         expect_status_details(200)
-        count = parsed_json.length
+        count = parsed_json["items"].size
 
         # ensure that the created fg is in returned fgs
         expect(count).to eq(old_count + 1)
-        expect(parsed_json[0].key?("id")).to be true
-        expect(parsed_json[0].key?("featurestoreName")).to be true
-        expect(parsed_json[0].key?("name")).to be true
-        expect(parsed_json[0]["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true
-        expect(parsed_json.any? { |fg| fg["name"] == featuregroup_name }).to be true
+        expect(parsed_json["items"][0].key?("id")).to be true
+        expect(parsed_json["items"][0].key?("featurestoreName")).to be true
+        expect(parsed_json["items"][0].key?("name")).to be true
+        expect(parsed_json["items"][0]["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true
+        expect(parsed_json["items"].any? { |fg| fg["name"] == featuregroup_name }).to be true
       end
 
       it "should be able to list all featuregroups of the project's featurestore sorted by name" do
@@ -3235,10 +3235,10 @@
         # get fgs
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc"
         expect_status_details(200)
-        names = json_body.map { |o| "#{o[:name]}" }
+        names = json_body[:items].map { |o| "#{o[:name]}" }
         sorted_names = names.sort_by(&:downcase)
 
-        expect(json_body.length).to eq(6)
+        expect(json_body[:items].size).to eq(6)
         expect(names).to eq(sorted_names)
       end
 
@@ -3249,10 +3249,11 @@
         # get fgs
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc,VERSION:asc"
         expect_status_details(200)
-        names_versions = json_body.map { |o| "#{o[:name]}_#{o[:version]}" }
+        names_versions = json_body[:items].map { |o| "#{o[:name]}_#{o[:version]}" }
         sorted_names_versions = names_versions.sort_by(&:downcase)
 
-        expect(json_body.length).to eq(6)
+        expect(json_body[:items].size).to eq(6)
+        expect(json_body[:count]).to eq(6)
         expect(names_versions).to eq(sorted_names_versions)
       end
 
@@ -3264,8 +3265,22 @@
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=NAME:featuregroup1"
         expect_status_details(200)
 
-        expect(json_body.length).to eq(3)
-        expect(json_body.all? { |fg| fg[:name] == "featuregroup1" }).to be true
+        expect(json_body[:items].size).to eq(3)
+        expect(json_body[:count]).to eq(3)
+        expect(json_body[:items].all? { |fg| fg[:name] == "featuregroup1" }).to be true
+      end
+
+      it "should be able to list all featuregroups of the project's featurestore filtered by name like" do
+        project = get_project
+        featurestore_id = get_featurestore_id(project.id)
+
+        # get fgs
+        get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=NAME_LIKE:group1"
+        expect_status_details(200)
+
+        expect(json_body[:items].size).to eq(3)
+        expect(json_body[:count]).to eq(3)
+        expect(json_body[:items].all? { |fg| fg[:name] == "featuregroup1" }).to be true
       end
 
       it "should be able to list all featuregroups of the project's featurestore filtered by latest_version" do
@@ -3275,11 +3290,12 @@
         # get fgs
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=latest_version"
         expect_status_details(200)
-        names = json_body.map { |o| "#{o[:name]}" }
+        names = json_body[:items].map { |o| "#{o[:name]}" }
 
-        expect(json_body.length).to eq(3)
-        expect(json_body.length).to eq(names.uniq.size)
-        expect(json_body.any? { |fg| fg[:name] == "featuregroup1" && fg[:version] == 3}).to be true
+        expect(json_body[:items].size).to eq(4)
+        expect(json_body[:count]).to eq(4)
+        expect(json_body[:items].size).to eq(names.uniq.size)
+        expect(json_body[:items].any? { |fg| fg[:name] == "featuregroup1" && fg[:version] == 3}).to be true
       end
 
       it "should be able to list all featuregroups of the project's featurestore filtered by name and version" do
@@ -3290,9 +3306,9 @@
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?filter_by=NAME:featuregroup1&filter_by=VERSION:2"
         expect_status_details(200)
 
-        expect(json_body.length).to eq(1)
-        expect(json_body.all? { |fg| fg[:name] == "featuregroup1" }).to be true
-        expect(json_body.all? { |fg| fg[:version] == 2 }).to be true
+        expect(json_body[:items].size).to eq(1)
+        expect(json_body[:items].all? { |fg| fg[:name] == "featuregroup1" }).to be true
+        expect(json_body[:items].all? { |fg| fg[:version] == 2 }).to be true
       end
 
       it "should be able to list all featuregroups of the project's featurestore limit" do
@@ -3303,7 +3319,8 @@
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc&limit=2"
         expect_status_details(200)
 
-        expect(json_body.length).to eq(2)
+        expect(json_body[:items].size).to eq(2)
+        expect(json_body[:count]).to eq(6)
       end
 
       it "should be able to list all featuregroups of the project's featurestore offset" do
@@ -3313,14 +3330,15 @@
         # get fgs
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc"
         expect_status_details(200)
-        names = json_body.map { |o| "#{o[:name]}" }
+        names = json_body[:items].map { |o| "#{o[:name]}" }
 
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups?sort_by=NAME:asc&offset=1"
         expect_status_details(200)
-        names_with_offset = json_body.map { |o| "#{o[:name]}" }
+        names_with_offset = json_body[:items].map { |o| "#{o[:name]}" }
 
-        expect(json_body.length).to eq(5)
-        for i in 0..json_body.length-1 do
+        expect(json_body[:items].size).to eq(5)
+        expect(json_body[:count]).to eq(6)
+        for i in 0..json_body[:items].size-1 do
           expect(names[i+1]).to eq(names_with_offset[i])
         end
       end

hopsworks-IT/src/test/ruby/spec/featureview_spec.rb

@@ -591,6 +591,7 @@
           expect_status_details(200)
 
           expect(parsed_json["items"].size).to eq(3)
+          expect(parsed_json["count"]).to eq(3)
         end
 
         it "should be able to get a list of feature view sorted by id" do
@@ -604,6 +605,7 @@
           sorted_ids = ids.sort
 
           expect(json_body[:items].length).to eq(3)
+          expect(json_body[:count]).to eq(3)
           expect(ids).to eq(ids.sort)
           expect(ids).not_to eq(ids.sort {|x, y| y <=> x})
         end
@@ -618,6 +620,7 @@
           ids = json_body[:items].map { |o| o[:id] }
 
           expect(json_body[:items].length).to eq(3)
+          expect(json_body[:count]).to eq(3)
           expect(ids).to eq(ids.sort {|x, y| y <=> x})
           expect(ids).not_to eq(ids.sort)
         end
@@ -633,6 +636,7 @@
           sorted_names_versions = names_versions.sort_by(&:downcase)
 
           expect(json_body[:items].length).to eq(3)
+          expect(json_body[:count]).to eq(3)
           expect(names_versions).to eq(sorted_names_versions)
         end
 
@@ -645,6 +649,20 @@
           expect_status_details(200)
 
           expect(json_body[:items].length).to eq(2)
+          expect(json_body[:count]).to eq(2)
+          expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true
+        end
+
+        it "should be able to get a list of feature view filtered by name like" do
+          project = get_project
+          featurestore_id = get_featurestore_id(project.id)
+
+          # Get the list
+          get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featureview?filter_by=NAME_LIKE:view2"
+          expect_status_details(200)
+
+          expect(json_body[:items].length).to eq(2)
+          expect(json_body[:count]).to eq(2)
           expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true
         end
 
@@ -658,6 +676,7 @@
           names = json_body[:items].map { |o| "#{o[:name]}" }
 
           expect(json_body[:items].length).to eq(2)
+          expect(json_body[:count]).to eq(2)
           expect(json_body[:items].length).to eq(names.uniq.size)
           expect(json_body[:items].any? { |fv| fv[:name] == "featureview2" && fv[:version] == 2}).to be true
         end
@@ -671,6 +690,7 @@
           expect_status_details(200)
 
           expect(json_body[:items].length).to eq(1)
+          expect(json_body[:count]).to eq(1)
           expect(json_body[:items].all? { |fv| fv[:name] == "featureview2" }).to be true
           expect(json_body[:items].all? { |fv| fv[:version] == 2 }).to be true
         end
@@ -684,6 +704,7 @@
           expect_status_details(200)
 
           expect(json_body[:items].length).to eq(2)
+          expect(json_body[:count]).to eq(3)
         end
 
         it "should be able to get a list of feature view offset" do
@@ -700,6 +721,7 @@
           ids_with_offset = json_body[:items].map { |o| o[:id] }
 
           expect(json_body[:items].length).to eq(2)
+          expect(json_body[:count]).to eq(3)
           for i in 0..json_body[:items].length-1 do
             expect(ids[i+1]).to eq(ids_with_offset[i])
           end

hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb

@@ -40,7 +40,9 @@ def create_test_files
   def get_storage_connectors(project_id, featurestore_id, type)
     json_result = get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/"
     connectors = JSON.parse(json_result)
-    connectors.select{|c| c['storageConnectorType'].eql?(type)}.map { |c| c.with_indifferent_access }
+    unless connectors["items"].nil?
+      connectors["items"].select{|c| c['storageConnectorType'].eql?(type)}.map { |c| c.with_indifferent_access }
+    end
   end
 
   def get_storage_connector(project_id, featurestore_id, name)

hopsworks-IT/src/test/ruby/spec/model_spec.rb

@@ -235,6 +235,16 @@
           expect(json_body[:items].count).to eq 3
           json_body[:items].each {|model| expect(model[:framework]).to eq "TENSORFLOW"}
         end
+        it "should get 4 models with version 1" do
+          get_models(@project[:id], "?filter_by=version:1")
+          expect_status_details(200)
+          expect(json_body[:items].count).to eq 4
+        end
+        it "should get 4 models with latest_version" do
+          get_models(@project[:id], "?filter_by=latest_version")
+          expect_status_details(200)
+          expect(json_body[:items].count).to eq 4
+        end
         it "should get 1 Python model with name_eq model_python" do
           get_models(@project[:id], "?filter_by=name_eq:model_python")
           expect_status_details(200)

hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb

@@ -588,7 +588,7 @@
           json_result, connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources")
           expect_status_details(201)
 
-          json_result, connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources")
+          json_result, @connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources")
           expect_status_details(201)
         end
 
@@ -600,7 +600,8 @@
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors"
           expect_status_details(200)
 
-          expect(json_body.length).to eq(5)
+          expect(json_body[:items].size).to eq(4)
+          expect(json_body[:count]).to eq(4)
         end
 
         it "should be able to get a list of connectors in the featurestore sorted by id" do
@@ -610,10 +611,11 @@
           # Get the list
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc"
           expect_status_details(200)
-          ids = json_body.map { |o| o[:id] }
+          ids = json_body[:items].map { |o| o[:id] }
           sorted_ids = ids.sort
 
-          expect(json_body.length).to eq(5)
+          expect(json_body[:items].size).to eq(4)
+          expect(json_body[:count]).to eq(4)
           expect(ids).to eq(sorted_ids)
         end
 
@@ -624,10 +626,11 @@
           # Get the list
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=NAME:asc"
           expect_status_details(200)
-          names = json_body.map { |o| "#{o[:name]}" }
+          names = json_body[:items].map { |o| "#{o[:name]}" }
           sorted_names = names.sort_by(&:downcase)
 
-          expect(json_body.length).to eq(5)
+          expect(json_body[:items].size).to eq(4)
+          expect(json_body[:count]).to eq(4)
           expect(names).to eq(sorted_names)
         end
 
@@ -639,8 +642,35 @@
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?filter_by=TYPE:HOPSFS"
           expect_status_details(200)
 
-          expect(json_body.length).to eq(4)
-          expect(json_body.all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true
+          expect(json_body[:items].size).to eq(4)
+          expect(json_body[:count]).to eq(4)
+          expect(json_body[:items].all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true
+        end
+
+        it "should be able to get a list of connectors in the featurestore filtered by name" do
+          project = get_project
+          featurestore_id = get_featurestore_id(project.id)
+
+          # Get the list
+          get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?filter_by=NAME:" + @connector_name
+          expect_status_details(200)
+
+          expect(json_body[:items].size).to eq(1)
+          expect(json_body[:count]).to eq(1)
+          expect(json_body[:items].all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true
+        end
+
+        it "should be able to get a list of connectors in the featurestore filtered by name like" do
+          project = get_project
+          featurestore_id = get_featurestore_id(project.id)
+
+          # Get the list
+          get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?filter_by=NAME_LIKE:hopsfs_connector_"
+          expect_status_details(200)
+
+          expect(json_body[:items].size).to eq(3)
+          expect(json_body[:count]).to eq(3)
+          expect(json_body[:items].all? { |sc| sc[:storageConnectorType] == "HOPSFS" }).to be true
         end
 
         it "should be able to get a list of connectors in the featurestore limit" do
@@ -651,7 +681,8 @@
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc&limit=2"
           expect_status_details(200)
 
-          expect(json_body.length == 2).to be true
+          expect(json_body[:items].size == 2).to be true
+          expect(json_body[:count]).to eq(4)
         end
 
         it "should be able to get a list of connectors in the featurestore offset" do
@@ -661,14 +692,15 @@
           # Get the list
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc"
           expect_status_details(200)
-          ids = json_body.map { |o| o[:id] }
+          ids = json_body[:items].map { |o| o[:id] }
 
           get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors?sort_by=ID:asc&offset=1"
           expect_status_details(200)
-          ids_with_offset = json_body.map { |o| o[:id] }
+          ids_with_offset = json_body[:items].map { |o| o[:id] }
 
-          expect(json_body.length).to eq(4)
-          for i in 0..json_body.length-1 do
+          expect(json_body[:items].size).to eq(3)
+          expect(json_body[:count]).to eq(4)
+          for i in 0..json_body[:items].size-1 do
             expect(ids[i+1]).to eq(ids_with_offset[i])
           end
         end