From a96c8195b3a25cc8bfaeea42dc99e4f0568811e1 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 12 Oct 2023 21:26:00 +0200 Subject: [PATCH 1/6] Fix: detaching views from ontology (#48) * add detach a view from an ontology test * fix populate_from_params help to reset object value to empty * reset to master branch the api dependecies --- Gemfile | 4 +- Gemfile.lock | 85 ++++++++++--------- helpers/application_helper.rb | 3 +- .../controllers/test_ontologies_controller.rb | 26 ++++++ 4 files changed, 74 insertions(+), 44 deletions(-) diff --git a/Gemfile b/Gemfile index 5ff855ea..cc21c703 100644 --- a/Gemfile +++ b/Gemfile @@ -42,12 +42,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'development' +gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'master' gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'master' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 46b2a605..e2ea3509 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: d0ac992c88bd417f2f2137ba62934c3c41b6db7c + revision: f440ae855a217807fead1d20629a0f187997b973 branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -11,8 +11,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: cda6aff2338e2a2831e4e7bf716abdf8fa8483d2 - branch: development + revision: bd7154217438c3b9160e0e9b495c7c718b55fbf8 + branch: master specs: goo (0.0.2) addressable (~> 2.8) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: e9b708c40b2b22b935fb48d18ed19de8148fca35 - branch: development + revision: bf682ec9c1baa3e9e7eae2e5d095187b0f900bf7 + branch: master specs: ontologies_linked_data (0.0.1) activesupport @@ -103,16 +103,16 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.1) + addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) - airbrussh (1.4.1) + airbrussh (1.5.0) sshkit (>= 1.6.1, != 1.7.0) - backports (3.23.0) - bcrypt (3.1.18) + backports (3.24.1) + bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.1) + capistrano (3.17.3) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -125,7 +125,8 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.2.0) + concurrent-ruby (1.2.2) + connection_pool (2.4.1) cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -157,12 +158,12 @@ GEM faraday-patron (1.0.0) faraday-rack (1.0.0) faraday-retry (1.0.3) - ffi (1.15.5) + ffi (1.16.3) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.12.0) - google-apis-core (>= 0.9.1, < 2.a) - google-apis-core (0.11.0) + google-apis-analytics_v3 (0.13.0) + google-apis-core (>= 0.11.0, < 2.a) + google-apis-core (0.11.1) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -171,10 +172,9 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.3.0) + googleauth (1.8.1) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) - memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) @@ -191,9 +191,9 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.6.3) - jwt (2.7.0) + jwt (2.7.1) kgio (2.11.4) - libxml-ruby (4.0.0) + libxml-ruby (4.1.1) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) @@ -202,12 +202,11 @@ GEM net-imap net-pop net-smtp - memoist (0.16.2) method_source (1.0.0) - mime-types (3.4.1) + mime-types (3.5.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) - mini_mime (1.1.2) + mime-types-data (3.2023.1003) + mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) @@ -215,7 +214,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.3.4) + net-imap (0.4.1) date net-protocol net-pop (0.1.2) @@ -224,11 +223,11 @@ GEM timeout net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) - net-smtp (0.3.3) + net-smtp (0.4.0) net-protocol - net-ssh (7.0.1) + net-ssh (7.2.0) netrc (0.11.0) - newrelic_rpm (8.16.0) + newrelic_rpm (9.5.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -239,37 +238,40 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.1) + public_suffix (5.0.3) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.6.1) rack (>= 1.0, < 3) - rack-cache (1.13.0) + rack-cache (1.14.0) rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) - rack-mini-profiler (3.0.0) + rack-mini-profiler (3.1.1) rack (>= 1.2.0) rack-protection (1.5.5) rack - rack-test (2.0.2) + rack-test (2.1.0) rack (>= 1.3) rack-timeout (0.6.3) - raindrops (0.20.0) + raindrops (0.20.1) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (4.8.1) + redis (5.0.7) + redis-client (>= 0.9.0) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) + redis-client (0.17.0) + connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) + redis-store (1.10.0) + redis (>= 4, < 6) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -280,7 +282,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.2.5) + rexml (3.2.6) rsolr (2.5.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) @@ -289,7 +291,7 @@ GEM rubyzip (2.3.2) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) - signet (0.17.0) + signet (0.18.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -316,13 +318,13 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.3) + sshkit (1.21.5) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.10.0) - tilt (2.0.11) - timeout (0.3.2) + temple (0.10.3) + tilt (2.3.0) + timeout (0.4.0) trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) @@ -341,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 6c44f25f..10871498 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -31,6 +31,7 @@ def populate_from_params(obj, params) # Deal with empty strings for String and URI empty_string = value.is_a?(String) && value.empty? old_string_value_exists = obj.respond_to?(attribute) && (obj.send(attribute).is_a?(String) || obj.send(attribute).is_a?(RDF::URI)) + old_string_value_exists = old_string_value_exists || (obj.respond_to?(attribute) && obj.send(attribute).is_a?(LinkedData::Models::Base)) if old_string_value_exists && empty_string value = nil elsif empty_string @@ -59,7 +60,7 @@ def populate_from_params(obj, params) # Replace the initial value with the object, handling Arrays as appropriate if value.is_a?(Array) value = value.map {|e| attr_cls.find(uri_as_needed(e)).include(attr_cls.attributes).first} - else + elsif !value.nil? value = attr_cls.find(uri_as_needed(value)).include(attr_cls.attributes).first end elsif attr_cls diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4713b699..b90f15e1 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -254,6 +254,32 @@ def test_download_acl_only end + def test_detach_a_view + view = Ontology.find(@@view_acronym).include(:viewOf).first + ont = view.viewOf + refute_nil view + refute_nil ont + + remove_view_of = {viewOf: ''} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(remove_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}" + onto = MultiJson.load(last_response.body) + assert_nil onto["viewOf"] + + + add_view_of = {viewOf: @@acronym} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(add_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}" + onto = MultiJson.load(last_response.body) + assert_equal onto["viewOf"], ont.id.to_s + end + private def check400(response) From d0667aa67a440fee448944750a4ca8750ff273dc Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Thu, 12 Oct 2023 22:01:41 +0200 Subject: [PATCH 2/6] [ontoportal-bot] Gemfile.lock update --- Gemfile | 10 ++++++---- Gemfile.lock | 30 +++++++++++++++++++----------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Gemfile b/Gemfile index cc21c703..dfc4fa69 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'activesupport', '~> 3.0' +gem 'activesupport', '~> 3.1' # see https://github.com/ncbo/ontologies_api/issues/69 gem 'bigdecimal', '1.4.2' gem 'faraday', '~> 1.9' @@ -18,7 +18,7 @@ gem 'sinatra-contrib', '~> 1.0' gem 'ffi' gem 'rack-accept', '~> 0.4' gem 'rack-attack', '~> 6.6.1', require: 'rack/attack' -gem 'rack-cache', '~> 1.0' +gem 'rack-cache', '~> 1.13.0' gem 'rack-cors', require: 'rack/cors' # GitHub dependency can be removed when https://github.com/niko/rack-post-body-to-params/pull/6 is merged and released gem 'rack-post-body-to-params', github: 'palexander/rack-post-body-to-params', branch: 'multipart_support' @@ -26,8 +26,9 @@ gem 'rack-timeout' gem 'redis-rack-cache', '~> 2.0' # Data access (caching) -gem 'redis' +gem 'redis', '~> 4.8.1' gem 'redis-activesupport' +gem 'redis-store', '1.9.1' # Monitoring gem 'cube-ruby', require: 'cube' @@ -71,4 +72,5 @@ group :test do gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io -end + gem 'webmock' +end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index e2ea3509..c7cde714 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,7 +126,8 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.2) - connection_pool (2.4.1) + crack (0.4.5) + rexml cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -181,6 +182,7 @@ GEM haml (5.2.2) temple (>= 0.8.0) tilt + hashdiff (1.0.1) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) @@ -244,7 +246,7 @@ GEM rack (>= 0.4) rack-attack (6.6.1) rack (>= 1.0, < 3) - rack-cache (1.14.0) + rack-cache (1.13.0) rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) @@ -260,22 +262,21 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.0.7) - redis-client (>= 0.9.0) + redis (4.8.1) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) - redis-client (0.17.0) - connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.10.0) - redis (>= 4, < 6) + redis-store (1.9.1) + redis (>= 4, < 5) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) + request_store (1.5.1) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) @@ -340,6 +341,10 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.19.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) webrick (1.8.1) PLATFORMS @@ -347,7 +352,7 @@ PLATFORMS x86_64-linux DEPENDENCIES - activesupport (~> 3.0) + activesupport (~> 3.1) bcrypt_pbkdf (>= 1.0, < 2.0) bigdecimal (= 1.4.2) capistrano (~> 3) @@ -375,7 +380,7 @@ DEPENDENCIES rack rack-accept (~> 0.4) rack-attack (~> 6.6.1) - rack-cache (~> 1.0) + rack-cache (~> 1.13.0) rack-cors rack-mini-profiler rack-post-body-to-params! @@ -383,9 +388,11 @@ DEPENDENCIES rack-timeout rake (~> 10.0) redcarpet - redis + redis (~> 4.8.1) redis-activesupport redis-rack-cache (~> 2.0) + redis-store (= 1.9.1) + request_store shotgun! simplecov simplecov-cobertura @@ -395,6 +402,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer + webmock BUNDLED WITH 2.3.23 From 1b4c7eb1836836d56d1be96d62075935defd35e1 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Thu, 12 Oct 2023 22:38:10 +0200 Subject: [PATCH 3/6] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c7cde714..e6ca88b0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -275,8 +275,6 @@ GEM declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) - request_store (1.5.1) - rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) @@ -348,7 +346,6 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES @@ -392,7 +389,6 @@ DEPENDENCIES redis-activesupport redis-rack-cache (~> 2.0) redis-store (= 1.9.1) - request_store shotgun! simplecov simplecov-cobertura From 79dad08d195dcdf70df02c07a94fab2fd0cd5a63 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Tue, 17 Oct 2023 09:00:44 +0200 Subject: [PATCH 4/6] Fix: isInActiveScheme attribute in the class/tree endpoint (#50) * add class tree test with scheme and collection filters * make the class tree endpoit load isInActiveScheme and isInActiveCollection for the select class --- controllers/classes_controller.rb | 6 --- helpers/classes_helper.rb | 41 +++++++++++++-------- test/controllers/test_schemes_controller.rb | 17 +++++++++ 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/controllers/classes_controller.rb b/controllers/classes_controller.rb index c8e55bf8..d792c172 100644 --- a/controllers/classes_controller.rb +++ b/controllers/classes_controller.rb @@ -262,13 +262,7 @@ def includes_param_check end end - def concept_schemes - params["concept_schemes"]&.split(',') || [] - end - def concept_collections - params["concept_collections"]&.split(',') || [] - end def request_display(attrs) diff --git a/helpers/classes_helper.rb b/helpers/classes_helper.rb index fa6c48cf..60becb22 100644 --- a/helpers/classes_helper.rb +++ b/helpers/classes_helper.rb @@ -32,23 +32,19 @@ def get_class(submission, load_attrs=nil) load_children = load_attrs.delete :children load_has_children = load_attrs.delete :hasChildren - if !load_children + unless load_children load_children = load_attrs.select { |x| x.instance_of?(Hash) && x.include?(:children) } - - if load_children.length == 0 - load_children = nil - end - if !load_children.nil? - load_attrs = load_attrs.select { |x| !(x.instance_of?(Hash) && x.include?(:children)) } - end + load_children = nil if load_children.length == 0 + load_attrs = load_attrs.select { |x| !(x.instance_of?(Hash) && x.include?(:children)) } unless load_children.nil? end + cls_uri = notation_to_class_uri(submission) if cls_uri.nil? cls_uri = RDF::URI.new(params[:cls]) - if !cls_uri.valid? + unless cls_uri.valid? error 400, "The input class id '#{params[:cls]}' is not a valid IRI" end end @@ -62,23 +58,38 @@ def get_class(submission, load_attrs=nil) error 404, "Resource '#{params[:cls]}' not found in ontology #{submission.ontology.acronym} submission #{submission.submissionId}" end - unless load_has_children.nil? - cls.load_has_children - end - if !load_children.nil? + + extra_include = [] + + extra_include << :hasChildren if load_has_children + extra_include << :isInActiveScheme if load_attrs.include?(:inScheme) + extra_include << :isInActiveCollection if load_attrs.include?(:memberOf) + + cls.load_computed_attributes(to_load: extra_include , + options: {schemes: concept_schemes, collections: concept_collections}) + + + unless load_children.nil? LinkedData::Models::Class.partially_load_children( - [cls],500,cls.submission) + [cls], 500, cls.submission) unless load_has_children.nil? cls.children.each do |c| c.load_has_children end end end - return cls + cls end end + def concept_schemes + params["concept_schemes"]&.split(',') || [] + end + + def concept_collections + params["concept_collections"]&.split(',') || [] + end end end diff --git a/test/controllers/test_schemes_controller.rb b/test/controllers/test_schemes_controller.rb index ebabc42f..d4504aa3 100644 --- a/test/controllers/test_schemes_controller.rb +++ b/test/controllers/test_schemes_controller.rb @@ -61,4 +61,21 @@ def test_calls_not_found assert_equal 404, last_response.status end + + def test_class_tree + ont = Ontology.find('INRAETHES-0').include(:acronym).first + sub = ont.latest_submission + sub.bring_remaining + sub.uri = RDF::URI.new('http://opendata.inrae.fr/thesaurusINRAE/domainesINRAE') + sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first + sub.save + + cls = 'http://opendata.inrae.fr/thesaurusINRAE/d_6' + get "ontologies/INRAETHES-0/classes/#{CGI.escape(cls)}/tree" + + classes = MultiJson.load(last_response.body) + + refute_nil classes.select{|x| x['@id'].eql?(cls)}.first['isInActiveScheme'] + refute_nil classes.select{|x| x['@id'].eql?(cls)}.first['isInActiveCollection'] + end end From eb9103bdba650047ebb110e7bc2cd0b85780c329 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 17 Oct 2023 12:54:33 +0200 Subject: [PATCH 5/6] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e6ca88b0..63f2fc7c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: bf682ec9c1baa3e9e7eae2e5d095187b0f900bf7 + revision: 0d6632bb39c1d24a930fe088bc464c3a51d68c9b branch: master specs: ontologies_linked_data (0.0.1) From 12340962f57d9e7107e8a32ad4b903caff529fa5 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 19 Oct 2023 14:06:43 +0200 Subject: [PATCH 6/6] Merge to master: Release 2.3.2 - Submissions endpoint pagination and fixes (#52) * add get submission all including all properties test * extract and use submission_include_params where we use submission.bring * use retrieve_submissions helper in the :acronym/submissions endpoint --- Gemfile.lock | 4 +- controllers/admin_controller.rb | 4 +- controllers/ontologies_controller.rb | 19 +-- .../ontology_submissions_controller.rb | 38 +++--- helpers/access_control_helper.rb | 4 - helpers/application_helper.rb | 42 ++---- helpers/request_params_helper.rb | 97 ++++++++++++++ helpers/submission_helper.rb | 67 ++++++++++ .../controllers/test_ontologies_controller.rb | 4 +- .../test_ontology_submissions_controller.rb | 125 +++++++++++++++++- .../thesaurusINRAE_nouv_structure.rdf | 2 +- 11 files changed, 329 insertions(+), 77 deletions(-) create mode 100644 helpers/submission_helper.rb diff --git a/Gemfile.lock b/Gemfile.lock index 63f2fc7c..8d0a6681 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 0d6632bb39c1d24a930fe088bc464c3a51d68c9b + revision: f44f7baa96eb3ee10dfab4a8aca154161ba7dd89 branch: master specs: ontologies_linked_data (0.0.1) @@ -112,7 +112,7 @@ GEM bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.3) + capistrano (3.18.0) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) diff --git a/controllers/admin_controller.rb b/controllers/admin_controller.rb index 7ae6d800..747def93 100644 --- a/controllers/admin_controller.rb +++ b/controllers/admin_controller.rb @@ -68,7 +68,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil? check_last_modified(latest) - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + latest.bring(*submission_include_params) NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions) halt 204 end @@ -84,7 +84,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) end check_last_modified(latest) if latest - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + latest.bring(*submission_include_params) if latest reply(latest || {}) end diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index da1b748c..58518420 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -38,21 +38,12 @@ class OntologiesController < ApplicationController else latest = ont.latest_submission(status: :any) end - check_last_modified(latest) if latest - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) + if latest - if includes_param.first == :all - # Bring what we need to display all attr of the submission - latest.bring_remaining - latest.bring({:contact=>[:name, :email], - :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, - :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}) - else - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) - end + check_last_modified(latest) + latest.bring(*submission_include_params) end - #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + reply(latest || {}) end @@ -62,7 +53,7 @@ class OntologiesController < ApplicationController patch '/:acronym/latest_submission' do ont = Ontology.find(params["acronym"]).first error 422, "You must provide an existing `acronym` to patch" if ont.nil? - + submission = ont.latest_submission(status: :any) submission.bring(*OntologySubmission.attributes) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index cf55659d..0068a5f1 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -1,9 +1,15 @@ class OntologySubmissionsController < ApplicationController get "/submissions" do check_last_modified_collection(LinkedData::Models::OntologySubmission) - #using appplication_helper method - options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")} - reply retrieve_latest_submissions(options).values + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY") + } + subs = retrieve_latest_submissions(options) + subs = subs.values unless page? + # Force to show ontology reviews, notes and projects by default only for this request + LinkedData::Models::Ontology.serialize_default(*(LinkedData::Models::Ontology.hypermedia_settings[:serialize_default] + [:reviews, :notes, :projects])) + reply subs end ## @@ -19,22 +25,18 @@ class OntologySubmissionsController < ApplicationController ## # Display all submissions of an ontology get do - ont = Ontology.find(params["acronym"]).include(:acronym).first + ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) - if includes_param.first == :all - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) - ont.bring(submissions: [:released, :creationDate, :status, :submissionId, - {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]) - - ont.submissions.each do |sub| - sub.bring_remaining - end - else - ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param)) - end - reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId + check_access(ont) + options = { + also_include_views: true, + status: (params["include_status"] || "ANY"), + ontology: params["acronym"] + } + subs = retrieve_submissions(options) + + reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId end ## @@ -53,7 +55,7 @@ class OntologySubmissionsController < ApplicationController ont.bring(:submissions) ont_submission = ont.submission(params["ontology_submission_id"]) error 404, "`submissionId` not found" if ont_submission.nil? - ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + ont_submission.bring(*submission_include_params) reply ont_submission end diff --git a/helpers/access_control_helper.rb b/helpers/access_control_helper.rb index 1de3bee5..74416866 100644 --- a/helpers/access_control_helper.rb +++ b/helpers/access_control_helper.rb @@ -10,11 +10,7 @@ module AccessControlHelper def check_access(obj) return obj unless LinkedData.settings.enable_security if obj.is_a?(Enumerable) - if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on? - check_access(obj.first) - else filter_access(obj) - end else if obj.respond_to?(:read_restricted?) && obj.read_restricted? readable = obj.readable?(env["REMOTE_USER"]) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 10871498..172170fa 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -52,6 +52,10 @@ def populate_from_params(obj, params) value = is_arr ? value : [value] new_value = [] value.each do |cls| + if uri_as_needed(cls["ontology"]).nil? + new_value << cls + next + end sub = LinkedData::Models::Ontology.find(uri_as_needed(cls["ontology"])).first.latest_submission new_value << LinkedData::Models::Class.find(cls["class"]).in(sub).first end @@ -356,40 +360,16 @@ def replace_url_prefix(id) end def retrieve_latest_submissions(options = {}) - status = (options[:status] || "RDF").to_s.upcase - include_ready = status.eql?("READY") ? true : false - status = "RDF" if status.eql?("READY") - any = true if status.eql?("ANY") - include_views = options[:also_include_views] || false - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << :submissionStatus unless includes.include?(:submissionStatus) - if any - submissions_query = OntologySubmission.where - else - submissions_query = OntologySubmission.where(submissionStatus: [ code: status]) - end + submissions = retrieve_submissions(options) - submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views - submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] - end - submissions = submissions_query.include(includes).to_a - - # Figure out latest parsed submissions using all submissions - latest_submissions = {} + latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| - # To retrieve all metadata, but slow when a lot of ontologies - if includes_param.first == :all - sub.bring_remaining + unless page? + next if include_ready?(options) && !sub.ready? + next if sub.ontology.nil? + latest_submissions[sub.ontology.acronym] ||= sub + latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end - next if include_ready && !sub.ready? - next if sub.ontology.nil? - latest_submissions[sub.ontology.acronym] ||= sub - latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end latest_submissions end diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index e7ec091a..842ee0a7 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -13,6 +13,10 @@ def settings_params(klass) [attributes, page, size, order_by, bring_unmapped] end + def page? + !params[:page].nil? + end + def is_set?(param) !param.nil? && param != "" end @@ -25,6 +29,38 @@ def filter build_filter end + def apply_filters(object, query) + attributes_to_filter = object.attributes(:all).select{|x| params.keys.include?(x.to_s)} + filters = attributes_to_filter.map {|key| [key, params[key]&.split(',')]}.to_h + add_direct_filters(filters, query) + end + + def apply_submission_filters(query) + + filters = { + naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng], + hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS], + ontology_hasDomain_acronym: params[:hasDomain]&.split(',') , #%w[Crop Vue_francais], + ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], + isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], + hasFormalityLevel: params[:hasFormalityLevel]&.split(','), #["http://w3id.org/nkos/nkostype#thesaurus"], + ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + } + inverse_filters = { + status: params[:status], #"retired", + submissionStatus: params[:submissionStatus] #"RDF", + } + + query = add_direct_filters(filters, query) + + query = add_inverse_filters(inverse_filters, query) + + query = add_acronym_name_filters(query) + + add_order_by_patterns(query) + end + + def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) orders = (params["order"] || default_order.to_s).split(',') @@ -50,6 +86,67 @@ def bring_unmapped_to(page_data, sub, klass) end private + def extract_attr(key) + attr, sub_attr, sub_sub_attr = key.to_s.split('_') + + return attr.to_sym unless sub_attr + + return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr + + {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} + end + + def add_direct_filters(filters, query) + filters.each do |key, values| + attr = extract_attr(key) + next if Array(values).empty? + + filter = Goo::Filter.new(attr).regex(values.first) + values.drop(1).each do |v| + filter = filter.or(Goo::Filter.new(attr).regex(v)) + end + query = query.filter(filter) + end + query + end + + def add_inverse_filters(inverse_filters, query) + inverse_filters.each do |key, value| + attr = extract_attr(key) + next unless value + + filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") + query = query.filter(filter) + end + query + end + + def add_acronym_name_filters(query) + if params[:acronym] + filter = Goo::Filter.new(extract_attr(:ontology_acronym)).regex(params[:acronym]) + if params[:name] + filter.or(Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name])) + end + query = query.filter(filter) + elsif params[:name] + filter = Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name]) + query = query.filter(filter) + end + query + end + + def add_order_by_patterns(query) + if params[:order_by] + attr, sub_attr = params[:order_by].to_s.split('_') + if sub_attr + order_pattern = { attr.to_sym => { sub_attr.to_sym => (sub_attr.eql?("name") ? :asc : :desc) } } + else + order_pattern = { attr.to_sym => :desc } + end + query = query.order_by(order_pattern) + end + query + end def sort_order_item(param, order) [param.to_sym, order.to_sym] diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb new file mode 100644 index 00000000..07f82138 --- /dev/null +++ b/helpers/submission_helper.rb @@ -0,0 +1,67 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module SubmissionHelper + def submission_include_params + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + includes = OntologySubmission.goo_attrs_to_load(includes_param) + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} + end + + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} + includes << {:contact=>[:name, :email]} + end + includes + end + + def submission_attributes_all + out = [LinkedData::Models::OntologySubmission.embed_values_hash] + out << {:contact=>[:name, :email]} + out << {:ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, + :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType]} + + out + end + + def retrieve_submissions(options) + status = (options[:status] || "RDF").to_s.upcase + status = "RDF" if status.eql?("READY") + ontology_acronym = options[:ontology] + any = status.eql?("ANY") + include_views = options[:also_include_views] || false + includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) + includes << :submissionStatus unless includes.include?(:submissionStatus) + + submissions_query = LinkedData::Models::OntologySubmission + submissions_query = submissions_query.where(ontology: [acronym: ontology_acronym]) if ontology_acronym + + if any + submissions_query = submissions_query.where unless ontology_acronym + else + submissions_query = submissions_query.where({ submissionStatus: [code: status] }) + end + + submissions_query = apply_submission_filters(submissions_query) + submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views + submissions_query = submissions_query.filter(filter) if filter? + + + submissions = submissions_query.include(submission_include_params) + if page? + submissions.page(page, size).all + else + submissions.to_a + end + end + + def include_ready?(options) + options[:status] && options[:status].to_s.upcase.eql?("READY") + end + + end + end +end + +helpers Sinatra::Helpers::SubmissionHelper \ No newline at end of file diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index b90f15e1..34f8c4dc 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -217,13 +217,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test1@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test2@example.org", password: "12345" }) blocked_user.save diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 7500dce4..8c4cb098 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,10 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}] + contact: [{name: "test_name", email: "test3@example.org"}], + URI: 'https://test.com/test', + status: 'production', + description: 'ontology description' } @@status_uploaded = "UPLOADED" @@status_rdf = "RDF" @@ -36,6 +39,12 @@ def self._create_onts ont.save end + def setup + delete_ontologies_and_submissions + ont = Ontology.new(acronym: @@acronym, name: @@name, administeredBy: [@@user]) + ont.save + end + def test_submissions_for_given_ontology num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 1) ontology = created_ont_acronyms.first @@ -156,13 +165,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test4@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test5@example.org", password: "12345" }) blocked_user.save @@ -192,4 +201,114 @@ def test_download_acl_only end end + + + def test_submissions_default_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + + submission_default_attributes = LinkedData::Models::OntologySubmission.hypermedia_settings[:serialize_default].map(&:to_s) + + get("/submissions?display_links=false&display_context=false&include_status=ANY") + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + end + + def test_submissions_all_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + def submission_all_attributes + attrs = OntologySubmission.goo_attrs_to_load([:all]) + embed_attrs = attrs.select { |x| x.is_a?(Hash) }.first + + attrs.delete_if { |x| x.is_a?(Hash) }.map(&:to_s) + embed_attrs.keys.map(&:to_s) + end + get("/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_custom_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + include = 'ontology,contact,submissionId' + + get("/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_param_include + skip('only for local development regrouping a set of tests') + test_submissions_default_includes + test_submissions_all_includes + test_submissions_custom_includes + end + + private + def submission_keys(sub) + sub.to_hash.keys - %w[@id @type id] + end end diff --git a/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf b/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf index 8353d82f..ca303834 100644 --- a/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf +++ b/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf @@ -30,7 +30,7 @@ 1331561625299 - aktivite + aktivite 2012-03-12T22:13:45Z 2017-09-22T14:09:06Z