From a3121099089120c40a938e2a97f6bf3f78c50f2c Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 6 Mar 2023 16:58:37 +0100 Subject: [PATCH 01/70] add request_lang middleware --- Gemfile | 3 ++- Gemfile.lock | 37 ++++++++++++++++++++++--------------- app.rb | 6 ++++++ lib/rack/request_lang.rb | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 lib/rack/request_lang.rb diff --git a/Gemfile b/Gemfile index 5ff855ea..c80ea2e9 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,7 @@ gem 'rake', '~> 10.0' gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' +gem 'request_store' # Rack middleware gem 'ffi' @@ -47,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/support-multilingual-read-one-language-from-request-parameter' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index d02647ba..ba50a063 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: cda6aff2338e2a2831e4e7bf716abdf8fa8483d2 + revision: b769c165906163e30a026dba511ae1069c4eed3d branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f4395ea6b32df8cee25b3ccf8515c9c75d18b19a - branch: development + revision: f8c8f569cc282ec31ad181cdbe355f7899983b3f + branch: feature/support-multilingual-read-one-language-from-request-parameter specs: ontologies_linked_data (0.0.1) activesupport @@ -112,7 +112,7 @@ GEM bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.1) + capistrano (3.17.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -125,7 +125,8 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.2.0) + concurrent-ruby (1.2.2) + connection_pool (2.3.0) cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -160,8 +161,8 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.12.0) - google-apis-core (>= 0.9.1, < 2.a) + google-apis-analytics_v3 (0.13.0) + google-apis-core (>= 0.11.0, < 2.a) google-apis-core (0.11.0) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) @@ -206,7 +207,7 @@ GEM method_source (1.0.0) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) + mime-types-data (3.2023.0218.1) mini_mime (1.1.2) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -228,7 +229,7 @@ GEM net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.16.0) + newrelic_rpm (9.0.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -256,24 +257,29 @@ GEM rack-test (2.0.2) rack (>= 1.3) rack-timeout (0.6.3) - raindrops (0.20.0) + raindrops (0.20.1) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (4.8.1) + redis (5.0.6) + redis-client (>= 0.9.0) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) + redis-client (0.13.0) + connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) + redis-store (1.9.2) + redis (>= 4, < 6) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) + request_store (1.5.1) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) @@ -316,12 +322,12 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.3) + sshkit (1.21.4) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) temple (0.10.0) - tilt (2.0.11) + tilt (2.1.0) timeout (0.3.2) trailblazer-option (0.1.2) tzinfo (2.0.6) @@ -384,6 +390,7 @@ DEPENDENCIES redis redis-activesupport redis-rack-cache (~> 2.0) + request_store shotgun! simplecov simplecov-cobertura diff --git a/app.rb b/app.rb index 5360ae4b..46457c86 100644 --- a/app.rb +++ b/app.rb @@ -29,6 +29,7 @@ require_relative 'lib/rack/cube_reporter' require_relative 'lib/rack/param_translator' require_relative 'lib/rack/slice_detection' +require_relative 'lib/rack/request_lang' # Logging setup require_relative "config/logging" @@ -36,6 +37,8 @@ # Inflector setup require_relative "config/inflections" +require 'request_store' + # Protection settings set :protection, :except => :path_traversal @@ -143,6 +146,9 @@ use Rack::PostBodyToParams use Rack::ParamTranslator +use RequestStore::Middleware +use Rack::RequestLang + use LinkedData::Security::Authorization use LinkedData::Security::AccessDenied diff --git a/lib/rack/request_lang.rb b/lib/rack/request_lang.rb new file mode 100644 index 00000000..b2221041 --- /dev/null +++ b/lib/rack/request_lang.rb @@ -0,0 +1,16 @@ +module Rack + class RequestLang + + def initialize(app = nil, options = {}) + @app = app + end + + def call(env) + r = Rack::Request.new(env) + lang = r.params["lang"] || r.params["language"] + lang = lang.upcase.to_sym if lang + RequestStore.store[:requested_lang] = lang + @app.call(env) + end + end +end \ No newline at end of file From 207884bede33c54e764117843843220d26244674 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 6 Mar 2023 19:17:40 +0100 Subject: [PATCH 02/70] pin redis gem version to 4.8.1 --- Gemfile | 2 +- Gemfile.lock | 82 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/Gemfile b/Gemfile index 82282bee..1737403e 100644 --- a/Gemfile +++ b/Gemfile @@ -26,7 +26,7 @@ gem 'rack-timeout' gem 'redis-rack-cache', '~> 2.0' # Data access (caching) -gem 'redis' +gem 'redis', '~> 4.8.1' gem 'redis-activesupport' # Monitoring diff --git a/Gemfile.lock b/Gemfile.lock index 27deaffe..e423a14a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 077c674a6e277a51dca4ca681e49e3e3a55b918a + revision: 919c20dec58375eb8a4dd1aed47864e2ad7bacfa branch: master specs: goo (0.0.2) @@ -15,7 +15,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 71d41e3afb35dafe29abfb6d9becaadc725bad36 + revision: 964f0680799421ab24eddc974d9f2995c6c88734 branch: master specs: ncbo_annotator (0.0.1) @@ -26,7 +26,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: a1a4babc0b5e36325b5af667a6a50b7af1f5f891 + revision: c09d76eeb131caf422c18ba49580db1b47df86dc branch: master specs: ncbo_cron (0.0.1) @@ -42,7 +42,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 6010ff60b99dc1282822b8a1fb59bd59f453755f + revision: d0ac992c88bd417f2f2137ba62934c3c41b6db7c branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 4f9139d870c3b1771af1127afa17b679bd0f60dc + revision: 546b3a45a6f359c09f0cc95d954f03e979b4847a branch: master specs: ontologies_linked_data (0.0.1) @@ -112,7 +112,7 @@ GEM bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.1) + capistrano (3.17.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -125,15 +125,16 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.2) cube-ruby (0.0.3) dante (0.2.0) + date (3.3.3) declarative (0.0.20) docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) ed25519 (1.3.0) - faraday (1.10.2) + faraday (1.10.3) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -159,9 +160,9 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.12.0) - google-apis-core (>= 0.9.1, < 2.a) - google-apis-core (0.9.1) + google-apis-analytics_v3 (0.13.0) + google-apis-core (>= 0.11.0, < 2.a) + google-apis-core (0.11.0) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -186,36 +187,48 @@ GEM httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.6.2) + json (2.6.3) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.6.2) - jwt (2.5.0) + json_pure (2.6.3) + jwt (2.7.0) kgio (2.11.4) - libxml-ruby (3.2.4) - logger (1.5.1) + libxml-ruby (4.0.0) + logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) - mail (2.7.1) + mail (2.8.1) mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp memoist (0.16.2) method_source (1.0.0) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) + mime-types-data (3.2023.0218.1) mini_mime (1.1.2) minitest (4.7.5) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - multipart-post (2.2.3) + multipart-post (2.3.0) net-http-persistent (2.9.4) + net-imap (0.3.4) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.1) + timeout net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) + net-smtp (0.3.3) + net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.13.1) + newrelic_rpm (9.0.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -223,10 +236,10 @@ GEM parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) - pry (0.14.1) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.0) + public_suffix (5.0.1) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -243,20 +256,20 @@ GEM rack-test (2.0.2) rack (>= 1.3) rack-timeout (0.6.3) - raindrops (0.20.0) + raindrops (0.20.1) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) - redcarpet (3.5.1) - redis (4.8.0) + redcarpet (3.6.0) + redis (4.8.1) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) + redis-store (1.9.2) + redis (>= 4, < 6) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -281,7 +294,7 @@ GEM faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) @@ -303,14 +316,15 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.3) + sshkit (1.21.4) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.9.1) - tilt (2.0.11) + temple (0.10.0) + tilt (2.1.0) + timeout (0.3.2) trailblazer-option (0.1.2) - tzinfo (2.0.5) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) unf (0.1.4) @@ -324,7 +338,7 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) - webrick (1.7.0) + webrick (1.8.1) PLATFORMS x86_64-darwin-21 @@ -367,7 +381,7 @@ DEPENDENCIES rack-timeout rake (~> 10.0) redcarpet - redis + redis (~> 4.8.1) redis-activesupport redis-rack-cache (~> 2.0) shotgun! From ae23ce082cab23158d0f038e9a7d956776d14376 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 6 Mar 2023 19:31:15 +0100 Subject: [PATCH 03/70] update Gemfile.lock --- Gemfile | 2 +- Gemfile.lock | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index b8371e74..71cdf420 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/support-multilingual-read-one-language-from-request-parameter' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 449708f4..38c87943 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f8c8f569cc282ec31ad181cdbe355f7899983b3f - branch: feature/support-multilingual-read-one-language-from-request-parameter + revision: 69756f8a7cff39d283065217559c68820d6d95e3 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -126,7 +126,6 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.2) - connection_pool (2.3.0) cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -262,13 +261,10 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.0.6) - redis-client (>= 0.9.0) + redis (4.8.1) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) - redis-client (0.13.0) - connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) From be31e4f6dd9b3a3d1a29d99eb7087367e731e45f Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 7 Mar 2023 18:18:16 +0000 Subject: [PATCH 04/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3d5d0c97..e29b281b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -107,7 +107,7 @@ GEM public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) - backports (3.23.0) + backports (3.24.0) bcrypt (3.1.18) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) @@ -342,9 +342,7 @@ GEM macaddr (~> 1.0) webrick (1.8.1) - PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES From 77bde061c596e22cf68aab477e6d1eb1270c1fd7 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 7 Mar 2023 18:34:47 +0000 Subject: [PATCH 05/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e29b281b..f176a828 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: b769c165906163e30a026dba511ae1069c4eed3d + revision: 597436e366417be0ec523c65d97e18f94e906b73 branch: development specs: goo (0.0.2) From 0d8e4fdcf64a9e2475104b5783987b2dac2b5623 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 7 Mar 2023 18:42:30 +0000 Subject: [PATCH 06/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f176a828..4b3f0d6d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 69756f8a7cff39d283065217559c68820d6d95e3 + revision: 4609ce3101bad8b9507488e50da92eb23c8a92ab branch: development specs: ontologies_linked_data (0.0.1) From aed3bf580e56b45beaf921a8dcbd6520f685b1f1 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 20 Mar 2023 06:07:11 +0100 Subject: [PATCH 07/70] update Gemfile to use ontologies_linked_data new metadata branch --- Gemfile | 2 +- Gemfile.lock | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index 71cdf420..da2c84f9 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/migrate-attrbiutes-metadata-to-scheme-file' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 4b3f0d6d..9613a47f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 597436e366417be0ec523c65d97e18f94e906b73 + revision: f9b73ce6b6fac92bffd0a2a1b9dc9ba9266acf79 branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 4609ce3101bad8b9507488e50da92eb23c8a92ab - branch: development + revision: 18c1daaf0fdcdfced75a062a70fe9796034d40d9 + branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) activesupport @@ -226,7 +226,7 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-smtp (0.3.3) net-protocol - net-ssh (7.0.1) + net-ssh (7.1.0) netrc (0.11.0) newrelic_rpm (9.0.0) oj (2.18.5) @@ -253,7 +253,7 @@ GEM rack (>= 1.2.0) rack-protection (1.5.5) rack - rack-test (2.0.2) + rack-test (2.1.0) rack (>= 1.3) rack-timeout (0.6.3) raindrops (0.20.1) @@ -343,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES From f0f56a68317f2ad29df406a88c62cb55726f3625 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 20 Mar 2023 06:42:49 +0100 Subject: [PATCH 08/70] update TestOntologySubmissionsController to be adapted to the new model --- test/controllers/test_ontology_submissions_controller.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 7500dce4..253fef54 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,10 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}] + contact: [{name: "test_name", email: "test@example.org"}], + URI: 'https://test.com/test', + status: 'production', + description: 'ontology description' } @@status_uploaded = "UPLOADED" @@status_rdf = "RDF" From 535e96e1aa133af6cd6527531e8e875f8454e725 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 20 Mar 2023 10:18:23 +0000 Subject: [PATCH 09/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9613a47f..566e61d5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 18c1daaf0fdcdfced75a062a70fe9796034d40d9 + revision: 8da57befa2e7f4953ab74e74ebf87da1352ab06a branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) @@ -343,7 +343,6 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES From 7320f50aa9558dc5fbc26220c6f9d5b9c8c4014a Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 20 Mar 2023 10:27:06 +0000 Subject: [PATCH 10/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 566e61d5..d79d63fd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 8da57befa2e7f4953ab74e74ebf87da1352ab06a + revision: f2271662e6429d7878c66296ed855042251b0f01 branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) From 62f5831cbbf369bed5479ed06d3c804551335a44 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 21 Mar 2023 09:01:03 +0000 Subject: [PATCH 11/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d79d63fd..7b61090b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f2271662e6429d7878c66296ed855042251b0f01 + revision: 1e6886bec9c0c07bf5bfaad51e095a04c5bd0197 branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) From e259c1c6509c8329c6311355fddc6526c7050aee Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 21 Mar 2023 13:33:32 +0100 Subject: [PATCH 12/70] in submission_metadata rename display with category --- helpers/metadata_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/metadata_helper.rb b/helpers/metadata_helper.rb index db61c414..7699e6a0 100644 --- a/helpers/metadata_helper.rb +++ b/helpers/metadata_helper.rb @@ -64,9 +64,9 @@ def klass_metadata(klass, type) # Get display from the metadata if klass.attribute_settings(attr)[:display].nil? - attr_settings[:display] = "no" + attr_settings[:category] = "no" else - attr_settings[:display] = klass.attribute_settings(attr)[:display] + attr_settings[:category] = klass.attribute_settings(attr)[:display] end if !klass.attribute_settings(attr)[:helpText].nil? From 2bf3ac8e43df39299e958a3b6a8e9a23ab8bbab8 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 21 Mar 2023 13:34:02 +0100 Subject: [PATCH 13/70] add to submission_metadata description and example fields --- helpers/metadata_helper.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/helpers/metadata_helper.rb b/helpers/metadata_helper.rb index 7699e6a0..2c5d7182 100644 --- a/helpers/metadata_helper.rb +++ b/helpers/metadata_helper.rb @@ -69,10 +69,18 @@ def klass_metadata(klass, type) attr_settings[:category] = klass.attribute_settings(attr)[:display] end - if !klass.attribute_settings(attr)[:helpText].nil? + unless klass.attribute_settings(attr)[:helpText].nil? attr_settings[:helpText] = klass.attribute_settings(attr)[:helpText] end + unless klass.attribute_settings(attr)[:description].nil? + attr_settings[:description] = klass.attribute_settings(attr)[:description] + end + + unless klass.attribute_settings(attr)[:example].nil? + attr_settings[:example] = klass.attribute_settings(attr)[:example] + end + attr_settings[:@context] = { "@vocab" => "#{id_url_prefix}metadata/" } From ab8c876b036f8db95848e79981561f759efa6e93 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 26 Apr 2023 09:21:47 +0200 Subject: [PATCH 14/70] add the option to do pagination for the submission endpoint --- controllers/ontology_submissions_controller.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index cf55659d..8749786d 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -1,9 +1,13 @@ class OntologySubmissionsController < ApplicationController get "/submissions" do check_last_modified_collection(LinkedData::Models::OntologySubmission) - #using appplication_helper method - options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")} - reply retrieve_latest_submissions(options).values + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY") + } + subs = retrieve_latest_submissions(options) + subs = subs.values unless page? + reply subs end ## From c6454e48f1854fb15aabedf2a2911179c118e91d Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 26 Apr 2023 09:22:25 +0200 Subject: [PATCH 15/70] extract retrieve_latest_submissions method to submission helper --- helpers/application_helper.rb | 38 ++++++-------------------- helpers/submission_helper.rb | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 helpers/submission_helper.rb diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 6c44f25f..84c26497 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -355,40 +355,18 @@ def replace_url_prefix(id) end def retrieve_latest_submissions(options = {}) - status = (options[:status] || "RDF").to_s.upcase - include_ready = status.eql?("READY") ? true : false - status = "RDF" if status.eql?("READY") - any = true if status.eql?("ANY") - include_views = options[:also_include_views] || false - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << :submissionStatus unless includes.include?(:submissionStatus) - if any - submissions_query = OntologySubmission.where - else - submissions_query = OntologySubmission.where(submissionStatus: [ code: status]) - end + submissions = retrieve_submissions(options) - submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views - submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] - end - submissions = submissions_query.include(includes).to_a - - # Figure out latest parsed submissions using all submissions - latest_submissions = {} + latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| # To retrieve all metadata, but slow when a lot of ontologies - if includes_param.first == :all - sub.bring_remaining + sub.bring_remaining if includes_param.first == :all + unless page? + next if include_ready?(options) && !sub.ready? + next if sub.ontology.nil? + latest_submissions[sub.ontology.acronym] ||= sub + latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end - next if include_ready && !sub.ready? - next if sub.ontology.nil? - latest_submissions[sub.ontology.acronym] ||= sub - latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end latest_submissions end diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb new file mode 100644 index 00000000..38108050 --- /dev/null +++ b/helpers/submission_helper.rb @@ -0,0 +1,51 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module SubmissionHelper + + def retrieve_submissions(options) + status = (options[:status] || "RDF").to_s.upcase + status = "RDF" if status.eql?("READY") + any = status.eql?("ANY") + include_views = options[:also_include_views] || false + includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) + includes << :submissionStatus unless includes.include?(:submissionStatus) + + + submissions_query = LinkedData::Models::OntologySubmission + if any + submissions_query = submissions_query.where + else + submissions_query = submissions_query.where({submissionStatus: [ code: status]}) + end + + submissions_query = apply_filters(submissions_query) + submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views + submissions_query = submissions_query.filter(filter) if filter? + + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + if includes_param.first == :all + includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain]} + end + + submissions = submissions_query.include(includes) + if page? + submissions.page(page, size).all + else + submissions.to_a + end + end + + def include_ready?(options) + options[:status] && options[:status].to_s.upcase.eql?("READY") + end + + end + end +end + +helpers Sinatra::Helpers::SubmissionHelper \ No newline at end of file From f1b1234644f394fa902730df82180114f6e1996c Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 26 Apr 2023 09:23:13 +0200 Subject: [PATCH 16/70] implement apply_filters to submissions endpoint using SPARQL FILTERs --- helpers/request_params_helper.rb | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index e7ec091a..251af299 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -13,6 +13,10 @@ def settings_params(klass) [attributes, page, size, order_by, bring_unmapped] end + def page? + !params[:page].nil? + end + def is_set?(param) !param.nil? && param != "" end @@ -25,6 +29,53 @@ def filter build_filter end + def apply_filters(query) + + filters = { + naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng], + hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS], + ontology_hasDomain_acronym: params[:hasDomain]&.split(',') , #%w[Crop Vue_francais], + ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], + ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), + isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], + viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + } + inverse_filters = { + status: params[:status], #"retired", + submissionStatus: params[:submissionStatus] #"RDF", + } + + filters.each do |key , values| + attr = extract_attr(key) + next if Array(values).empty? + + filter = Goo::Filter.new(attr).regex(values.first) + values.drop(1).each do |v| + filter = filter.or(Goo::Filter.new(attr).regex(v)) + end + query = query.filter(filter) + end + + inverse_filters.each do |key ,value| + attr = extract_attr(key) + next unless value + + filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") + query = query.filter(filter) + end + query + end + + def extract_attr(key) + attr, sub_attr, sub_sub_attr = key.to_s.split('_') + + return attr.to_sym unless sub_attr + + return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr + + {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} + end + def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) orders = (params["order"] || default_order.to_s).split(',') From 64e7286516e0d5b783f9c8746b42216608c493a6 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Fri, 28 Apr 2023 10:49:41 +0200 Subject: [PATCH 17/70] add test for submissions endpoint pagination --- Gemfile | 2 +- Gemfile.lock | 17 +++++++++-------- .../test_ontology_submissions_controller.rb | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index da2c84f9..71cdf420 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/migrate-attrbiutes-metadata-to-scheme-file' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 7b61090b..d76badc2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: f9b73ce6b6fac92bffd0a2a1b9dc9ba9266acf79 + revision: 1484ad56e39810208ee08a14d1319b6bc112f647 branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 1e6886bec9c0c07bf5bfaad51e095a04c5bd0197 - branch: feature/migrate-attrbiutes-metadata-to-scheme-file + revision: f2168c8ad9ec447338da914d10a352df558494b5 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -103,11 +103,11 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.1) + addressable (2.8.4) public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) - backports (3.24.0) + backports (3.24.1) bcrypt (3.1.18) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) @@ -171,7 +171,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.3.0) + googleauth (1.5.2) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.0.0) + newrelic_rpm (9.2.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -249,7 +249,7 @@ GEM rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) - rack-mini-profiler (3.0.0) + rack-mini-profiler (3.1.0) rack (>= 1.2.0) rack-protection (1.5.5) rack @@ -343,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 253fef54..ee8576bd 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -195,4 +195,19 @@ def test_download_acl_only end end + def test_submissions_pagination + num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 2, submission_count: 2) + + get "/submissions" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal 2, submissions.length + + + get "/submissions?page=1&pagesize=1" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + end end From 7e86f58e4ac1a2de75dc2cd269375cbe3f38c9d1 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Fri, 28 Apr 2023 09:20:55 +0000 Subject: [PATCH 18/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d76badc2..152465f7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -343,7 +343,6 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES From 1b7d514d949ce50bce74440082b83048b3a3aeec Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 00:43:39 +0200 Subject: [PATCH 19/70] fix private only submission filter --- helpers/request_params_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 251af299..30c4bf43 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -38,7 +38,7 @@ def apply_filters(query) ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], - viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] } inverse_filters = { status: params[:status], #"retired", From 4cb42d0a333d5a57da6780e7a90fa8448f732cfc Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 17:56:54 +0200 Subject: [PATCH 20/70] add hasFormalityLevel filter for submissions endpoint --- helpers/request_params_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 30c4bf43..3f973dec 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -38,6 +38,7 @@ def apply_filters(query) ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], + hasFormalityLevel: params[:hasFormalityLevel]&.split(','), #["http://w3id.org/nkos/nkostype#thesaurus"], ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] } inverse_filters = { From 864e42fd8b8deffc6bc8c9e74765a1643026ee0a Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 17:31:54 +0200 Subject: [PATCH 21/70] add for ontology: reviews, notes, projects on the submissions endpoints --- controllers/ontology_submissions_controller.rb | 2 ++ helpers/submission_helper.rb | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 8749786d..9dd0d241 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -7,6 +7,8 @@ class OntologySubmissionsController < ApplicationController } subs = retrieve_latest_submissions(options) subs = subs.values unless page? + # Force to show ontology reviews, notes and projects by default only for this request + LinkedData::Models::Ontology.serialize_default(*(LinkedData::Models::Ontology.hypermedia_settings[:serialize_default] + [:reviews, :notes, :projects])) reply subs end diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 38108050..73550369 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -26,10 +26,12 @@ def retrieve_submissions(options) # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + includes = [:submissionId, {:contact=>[:name, :email], + :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain]} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} end submissions = submissions_query.include(includes) From 04971936b6468897e41e42046aa8174a4a214a13 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:01:37 +0200 Subject: [PATCH 22/70] bring submission metrics for submissions endpoints --- controllers/ontologies_controller.rb | 2 +- controllers/ontology_submissions_controller.rb | 2 +- helpers/submission_helper.rb | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index da1b748c..51b77cbd 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -47,7 +47,7 @@ class OntologiesController < ApplicationController latest.bring({:contact=>[:name, :email], :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}) + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) else latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) end diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 9dd0d241..69c2a4cd 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -32,7 +32,7 @@ class OntologySubmissionsController < ApplicationController # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) ont.bring(submissions: [:released, :creationDate, :status, :submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]) + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus], :metrics =>[:classes, :individuals, :properties]) ont.submissions.each do |sub| sub.bring_remaining diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 73550369..df141802 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -29,7 +29,8 @@ def retrieve_submissions(options) includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, + :submissionStatus] elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} end From 3abc4306a0ae0a26f46e00239552f884e3dc60f0 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:04:53 +0200 Subject: [PATCH 23/70] bring all contact attributes if asked in the submissions endpoints --- controllers/ontologies_controller.rb | 6 +++++- helpers/submission_helper.rb | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index 51b77cbd..1a8f2ebf 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -49,7 +49,11 @@ class OntologiesController < ApplicationController :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) else - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + includes = OntologySubmission.goo_attrs_to_load(includes_param) + + includes << {:contact=>[:name, :email]} if includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} + + latest.bring(*includes) end end #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index df141802..e19061c0 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -33,6 +33,8 @@ def retrieve_submissions(options) :submissionStatus] elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} + elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} + includes << {:contact=>[:name, :email]} end submissions = submissions_query.include(includes) From 69ef8bd9fb2fa268361d9f911a375e96db3c5671 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:08:56 +0200 Subject: [PATCH 24/70] refactor submissions endpoint filters by extracting some methods --- helpers/request_params_helper.rb | 77 +++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 3f973dec..d3ced82d 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -46,36 +46,11 @@ def apply_filters(query) submissionStatus: params[:submissionStatus] #"RDF", } - filters.each do |key , values| - attr = extract_attr(key) - next if Array(values).empty? - - filter = Goo::Filter.new(attr).regex(values.first) - values.drop(1).each do |v| - filter = filter.or(Goo::Filter.new(attr).regex(v)) - end - query = query.filter(filter) - end - - inverse_filters.each do |key ,value| - attr = extract_attr(key) - next unless value + query = add_direct_filters(filters, query) - filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") - query = query.filter(filter) - end - query + query = add_inverse_filters(inverse_filters, query) end - def extract_attr(key) - attr, sub_attr, sub_sub_attr = key.to_s.split('_') - - return attr.to_sym unless sub_attr - - return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr - - {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} - end def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) @@ -102,6 +77,54 @@ def bring_unmapped_to(page_data, sub, klass) end private + def extract_attr(key) + attr, sub_attr, sub_sub_attr = key.to_s.split('_') + + return attr.to_sym unless sub_attr + + return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr + + {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} + end + + def add_direct_filters(filters, query) + filters.each do |key, values| + attr = extract_attr(key) + next if Array(values).empty? + + filter = Goo::Filter.new(attr).regex(values.first) + values.drop(1).each do |v| + filter = filter.or(Goo::Filter.new(attr).regex(v)) + end + query = query.filter(filter) + end + query + end + + def add_inverse_filters(inverse_filters, query) + inverse_filters.each do |key, value| + attr = extract_attr(key) + next unless value + + filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") + query = query.filter(filter) + end + query + end + + def add_acronym_name_filters(query) + if params[:acronym] + filter = Goo::Filter.new(extract_attr(:ontology_acronym)).regex(params[:acronym]) + if params[:name] + filter.or(Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name])) + end + query = query.filter(filter) + elsif params[:name] + filter = Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name]) + query = query.filter(filter) + end + query + end def sort_order_item(param, order) [param.to_sym, order.to_sym] From 1655473ab592af03ee34e64dd8c1af92fbb18a95 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:09:48 +0200 Subject: [PATCH 25/70] add ontology acronym or name filters for submissions endpoints --- helpers/request_params_helper.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index d3ced82d..a9b52cc0 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -49,6 +49,8 @@ def apply_filters(query) query = add_direct_filters(filters, query) query = add_inverse_filters(inverse_filters, query) + + query = add_acronym_name_filters(query) end From 060ed269e78027b5d01d22ea2d2829e11d3c048f Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:10:34 +0200 Subject: [PATCH 26/70] add submissions endpoint order_by option --- Gemfile.lock | 11 +++++------ helpers/request_params_helper.rb | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d76badc2..5dda09cb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 1484ad56e39810208ee08a14d1319b6bc112f647 + revision: c3456c45c12ed92d4a3ae43cac7c1d4cdbf290b6 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f2168c8ad9ec447338da914d10a352df558494b5 + revision: 4ac9873f3d016259196a17cfc9c6ab8149468ec9 branch: development specs: ontologies_linked_data (0.0.1) @@ -193,7 +193,7 @@ GEM json_pure (2.6.3) jwt (2.7.0) kgio (2.11.4) - libxml-ruby (4.0.0) + libxml-ruby (4.1.0) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.2.0) + newrelic_rpm (9.2.1) oj (2.18.5) omni_logger (0.1.4) logger @@ -344,7 +344,6 @@ GEM PLATFORMS x86_64-darwin-21 - x86_64-linux DEPENDENCIES activesupport (~> 3.0) @@ -398,4 +397,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.14 + 2.3.23 diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index a9b52cc0..cf8ba1f8 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -51,6 +51,8 @@ def apply_filters(query) query = add_inverse_filters(inverse_filters, query) query = add_acronym_name_filters(query) + + add_order_by_patterns(query) end @@ -128,6 +130,19 @@ def add_acronym_name_filters(query) query end + def add_order_by_patterns(query) + if params[:order_by] + attr, sub_attr = params[:order_by].to_s.split('_') + if sub_attr + order_pattern = { attr.to_sym => { sub_attr.to_sym => (sub_attr.eql?("name") ? :asc : :desc) } } + else + order_pattern = { attr.to_sym => :desc } + end + query = query.order_by(order_pattern) + end + query + end + def sort_order_item(param, order) [param.to_sym, order.to_sym] end From 059d7e1bd1737dfff76479d86ade4e0dc81e26e1 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 1 May 2023 19:14:56 +0000 Subject: [PATCH 27/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 89a9d393..5e8cfa16 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -343,10 +343,8 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux - DEPENDENCIES activesupport (~> 3.0) bcrypt_pbkdf (>= 1.0, < 2.0) @@ -399,4 +397,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.23 + 2.3.14 From 3f29887a8236c17eae5cbf5f0b8c8bcf2615aea8 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 21:45:28 +0200 Subject: [PATCH 28/70] fix including ontology and contacts in the submissions endpoints --- helpers/submission_helper.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index e19061c0..6621ed43 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -31,12 +31,16 @@ def retrieve_submissions(options) :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, :submissionStatus] - elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} - elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} - includes << {:contact=>[:name, :email]} - end + else + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} + end + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} + includes << {:contact=>[:name, :email]} + end + end + submissions = submissions_query.include(includes) if page? submissions.page(page, size).all From b45618a53a09be206f9cff64fb35c8e1a0b5ff7f Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 23:34:19 +0200 Subject: [PATCH 29/70] fix list admin filter_access control (e.g for submissions endpoints) --- helpers/access_control_helper.rb | 4 ---- helpers/submission_helper.rb | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/helpers/access_control_helper.rb b/helpers/access_control_helper.rb index 1de3bee5..74416866 100644 --- a/helpers/access_control_helper.rb +++ b/helpers/access_control_helper.rb @@ -10,11 +10,7 @@ module AccessControlHelper def check_access(obj) return obj unless LinkedData.settings.enable_security if obj.is_a?(Enumerable) - if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on? - check_access(obj.first) - else filter_access(obj) - end else if obj.respond_to?(:read_restricted?) && obj.read_restricted? readable = obj.readable?(env["REMOTE_USER"]) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 6621ed43..0a082823 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -33,14 +33,14 @@ def retrieve_submissions(options) :submissionStatus] else if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl]} end if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} includes << {:contact=>[:name, :email]} end end - + submissions = submissions_query.include(includes) if page? submissions.page(page, size).all From b1e9ddf605bc7e232fe346b25f6f291fe094448e Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 1 May 2023 21:36:20 +0000 Subject: [PATCH 30/70] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5e8cfa16..1d512537 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.2.1) + newrelic_rpm (9.2.2) oj (2.18.5) omni_logger (0.1.4) logger From 34f19a3fd9f865f7e547d36275bba1ae6dd0229d Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 2 May 2023 00:35:56 +0200 Subject: [PATCH 31/70] check access of ontologies in /ontologies/:acronym/submissions endpoint --- controllers/ontology_submissions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 69c2a4cd..96411a8d 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -25,9 +25,10 @@ class OntologySubmissionsController < ApplicationController ## # Display all submissions of an ontology get do - ont = Ontology.find(params["acronym"]).include(:acronym).first + ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) + check_access(ont) if includes_param.first == :all # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) ont.bring(submissions: [:released, :creationDate, :status, :submissionId, From d5dcb9edcb972c29fe97d777575e2de4f79398ba Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 2 May 2023 03:58:27 +0200 Subject: [PATCH 32/70] include ontology viewOf attribute in the submission endpoints --- helpers/submission_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 0a082823..07e4f27a 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -33,7 +33,7 @@ def retrieve_submissions(options) :submissionStatus] else if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl]} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} end if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} From 57d1a1eb644776cb86e4ef8a2315948cf83ac2bc Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 15 Jun 2023 10:15:35 +0200 Subject: [PATCH 33/70] make apply_filters helper generic for any of model attributes --- helpers/request_params_helper.rb | 8 +++++++- helpers/submission_helper.rb | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index cf8ba1f8..45091042 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -29,7 +29,13 @@ def filter build_filter end - def apply_filters(query) + def apply_filters(object, query) + attributes_to_filter = object.attributes(:all).select{|x| params.keys.include?(x.to_s)} + filters = attributes_to_filter.map {|key| [key, params[key]&.split(',')]}.to_h + add_direct_filters(filters, query) + end + + def apply_submission_filters(query) filters = { naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng], diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 07e4f27a..534ddbd8 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -20,7 +20,7 @@ def retrieve_submissions(options) submissions_query = submissions_query.where({submissionStatus: [ code: status]}) end - submissions_query = apply_filters(submissions_query) + submissions_query = apply_submission_filters(submissions_query) submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views submissions_query = submissions_query.filter(filter) if filter? From dd26aac48c1c6f1420bb61535645cc5bb75fc2e1 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 15 Jun 2023 10:15:55 +0200 Subject: [PATCH 34/70] add Agents controller --- Gemfile | 2 +- Gemfile.lock | 19 ++++---- controllers/agents_controller.rb | 84 ++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 controllers/agents_controller.rb diff --git a/Gemfile b/Gemfile index 71cdf420..8326c2dd 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/add-persons-and-organization-models' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 1d512537..0ebac83d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: c3456c45c12ed92d4a3ae43cac7c1d4cdbf290b6 + revision: 39b7f2f2e9c398b981fb03feaa1eebc270877341 branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 4ac9873f3d016259196a17cfc9c6ab8149468ec9 - branch: development + revision: 6f72110402537fffe295c4bfd6c1fd458f343942 + branch: feature/add-persons-and-organization-models specs: ontologies_linked_data (0.0.1) activesupport @@ -112,7 +112,7 @@ GEM bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.2) + capistrano (3.17.3) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -191,9 +191,9 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.6.3) - jwt (2.7.0) + jwt (2.7.1) kgio (2.11.4) - libxml-ruby (4.1.0) + libxml-ruby (4.1.1) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) @@ -215,7 +215,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.3.4) + net-imap (0.3.6) date net-protocol net-pop (0.1.2) @@ -322,8 +322,8 @@ GEM net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.10.0) - tilt (2.1.0) + temple (0.10.2) + tilt (2.2.0) timeout (0.3.2) trailblazer-option (0.1.2) tzinfo (2.0.6) @@ -343,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb new file mode 100644 index 00000000..b6088b5c --- /dev/null +++ b/controllers/agents_controller.rb @@ -0,0 +1,84 @@ +class AgentsController < ApplicationController + + ## + # Ontology agents + get "/ontologies/:acronym/agents" do + end + + namespace "/agents" do + # Display all agents + get do + check_last_modified_collection(LinkedData::Models::Agent) + query = LinkedData::Models::Agent.where + query = apply_filters(LinkedData::Models::Agent, query) + reply query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).to_a + end + + # Display a single agent + get '/:id' do + check_last_modified_collection(LinkedData::Models::Agent) + id = params["id"] + agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first + error 404, "Agent #{id} not found" if agent.nil? + reply 200, agent + end + + # Create a agent with the given acronym + post do + create_agent + end + + # Create a agent with the given acronym + put '/:acronym' do + create_agent + end + + # Update an existing submission of a agent + patch '/:id' do + acronym = params["id"] + agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.attributes).first + + if agent.nil? + error 400, "Agent does not exist, please create using HTTP PUT before modifying" + else + populate_from_params(agent, params) + + if agent.valid? + agent.save + else + error 400, agent.errors + end + end + halt 204 + end + + # Delete a agent + delete '/:id' do + agent = LinkedData::Models::Agent.find(params["id"]).first + agent.delete + halt 204 + end + + private + + def create_agent + params ||= @params + acronym = params["id"] + agent = nil + agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if acronym + + if agent.nil? + agent = instance_from_params(LinkedData::Models::Agent, params) + else + error 400, "Agent exists, please use HTTP PATCH to update" + end + + if agent.valid? + agent.save + else + error 400, agent.errors + end + reply 201, agent + end + end +end \ No newline at end of file From 76db662a30e441774d4c8f41ed894f6d9b64c3f0 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 15 Jun 2023 11:41:21 +0200 Subject: [PATCH 35/70] add pagination to agents index endpoint if asked --- controllers/agents_controller.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index b6088b5c..d547441c 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -11,7 +11,14 @@ class AgentsController < ApplicationController check_last_modified_collection(LinkedData::Models::Agent) query = LinkedData::Models::Agent.where query = apply_filters(LinkedData::Models::Agent, query) - reply query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).to_a + query = query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)) + if page? + page, size = page_params + agents = query.page(page, size).all + else + agents = query.to_a + end + reply agents end # Display a single agent From 49d4ab4913f2016a424668cbe0da752f15e91cc1 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 15 Jun 2023 23:58:06 +0200 Subject: [PATCH 36/70] make agents routes work for /Agent and /agent --- controllers/agents_controller.rb | 141 +++++++++++++++---------------- 1 file changed, 70 insertions(+), 71 deletions(-) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index d547441c..e4a5d03f 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -1,91 +1,90 @@ class AgentsController < ApplicationController - ## - # Ontology agents - get "/ontologies/:acronym/agents" do - end - - namespace "/agents" do - # Display all agents - get do - check_last_modified_collection(LinkedData::Models::Agent) - query = LinkedData::Models::Agent.where - query = apply_filters(LinkedData::Models::Agent, query) - query = query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)) - if page? - page, size = page_params - agents = query.page(page, size).all - else - agents = query.to_a + %w[/agents /Agents].each do |namespace| + namespace namespace do + # Display all agents + get do + check_last_modified_collection(LinkedData::Models::Agent) + query = LinkedData::Models::Agent.where + query = apply_filters(LinkedData::Models::Agent, query) + query = query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)) + if page? + page, size = page_params + agents = query.page(page, size).all + else + agents = query.to_a + end + reply agents end - reply agents - end - - # Display a single agent - get '/:id' do - check_last_modified_collection(LinkedData::Models::Agent) - id = params["id"] - agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first - error 404, "Agent #{id} not found" if agent.nil? - reply 200, agent - end - # Create a agent with the given acronym - post do - create_agent - end + # Display a single agent + get '/:id' do + check_last_modified_collection(LinkedData::Models::Agent) + id = params["id"] + agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first + error 404, "Agent #{id} not found" if agent.nil? + reply 200, agent + end - # Create a agent with the given acronym - put '/:acronym' do - create_agent - end + # Create a agent with the given acronym + post do + create_agent + end - # Update an existing submission of a agent - patch '/:id' do - acronym = params["id"] - agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.attributes).first + # Create a agent with the given acronym + put '/:acronym' do + create_agent + end - if agent.nil? - error 400, "Agent does not exist, please create using HTTP PUT before modifying" - else - populate_from_params(agent, params) + # Update an existing submission of a agent + patch '/:id' do + acronym = params["id"] + agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.attributes).first - if agent.valid? - agent.save + if agent.nil? + error 400, "Agent does not exist, please create using HTTP PUT before modifying" else - error 400, agent.errors + populate_from_params(agent, params) + + if agent.valid? + agent.save + else + error 400, agent.errors + end end + halt 204 end - halt 204 - end - # Delete a agent - delete '/:id' do - agent = LinkedData::Models::Agent.find(params["id"]).first - agent.delete - halt 204 - end + # Delete a agent + delete '/:id' do + agent = LinkedData::Models::Agent.find(params["id"]).first + agent.delete + halt 204 + end - private + private - def create_agent - params ||= @params - acronym = params["id"] - agent = nil - agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if acronym + def create_agent + params ||= @params + acronym = params["id"] + agent = nil + agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if acronym - if agent.nil? - agent = instance_from_params(LinkedData::Models::Agent, params) - else - error 400, "Agent exists, please use HTTP PATCH to update" - end + if agent.nil? + agent = instance_from_params(LinkedData::Models::Agent, params) + else + error 400, "Agent exists, please use HTTP PATCH to update" + end - if agent.valid? - agent.save - else - error 400, agent.errors + if agent.valid? + agent.save + else + error 400, agent.errors + end + reply 201, agent end - reply 201, agent + end end + end \ No newline at end of file From 0ddc5de47103d617b46ada5ea2a95bdea4ba3172 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Fri, 16 Jun 2023 03:26:46 +0200 Subject: [PATCH 37/70] handle agent indentifiers and affiliations attributes save and update --- controllers/agents_controller.rb | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index e4a5d03f..9d8a984b 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -44,7 +44,12 @@ class AgentsController < ApplicationController if agent.nil? error 400, "Agent does not exist, please create using HTTP PUT before modifying" else + identifiers = params.delete "identifiers" + affiliations = params.delete "affiliations" + params.delete "id" populate_from_params(agent, params) + agent.identifiers = update_identifiers(identifiers) + agent.affiliations = update_affiliations(affiliations) if agent.valid? agent.save @@ -64,6 +69,48 @@ class AgentsController < ApplicationController private + def update_identifiers(identifiers) + Array(identifiers).map do |i| + identifier = LinkedData::Models::AgentIdentifier.find(i["notation"]).first + + if identifier + i.delete "id" + populate_from_params(identifier, i) + else + identifier = LinkedData::Models::AgentIdentifier.new(i) + end + + if identifier.valid? + identifier.save + else + error 400, identifier.errors + end + + identifier + end + end + + def update_affiliations(affiliations) + Array(affiliations).map do |aff| + affiliations = LinkedData::Models::Agent.find(aff["id"]) + + if affiliations + aff.delete "id" + populate_from_params(affiliations, aff) + else + affiliations = LinkedData::Models::Agent.new(aff) + end + + if affiliations.valid? + affiliations.save + else + error 400, affiliations.errors + end + + affiliations + end + end + def create_agent params ||= @params acronym = params["id"] @@ -71,7 +118,12 @@ def create_agent agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if acronym if agent.nil? + identifiers = params.delete "identifiers" + affiliations = params.delete "affiliations" + params.delete "id" agent = instance_from_params(LinkedData::Models::Agent, params) + agent.identifiers = update_identifiers(identifiers) + agent.affiliations = update_affiliations(affiliations) else error 400, "Agent exists, please use HTTP PATCH to update" end From f7c9bb9ce4689e00e5e8d9deffb38aba1a132aff Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 19 Jun 2023 01:10:09 +0200 Subject: [PATCH 38/70] make agent controller work for affiliations attribute --- controllers/agents_controller.rb | 85 ++++++++++++++------------------ 1 file changed, 36 insertions(+), 49 deletions(-) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index 9d8a984b..fa22586d 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -28,12 +28,12 @@ class AgentsController < ApplicationController # Create a agent with the given acronym post do - create_agent + reply 201, create_new_agent end # Create a agent with the given acronym put '/:acronym' do - create_agent + reply 201, create_new_agent end # Update an existing submission of a agent @@ -44,18 +44,9 @@ class AgentsController < ApplicationController if agent.nil? error 400, "Agent does not exist, please create using HTTP PUT before modifying" else - identifiers = params.delete "identifiers" - affiliations = params.delete "affiliations" - params.delete "id" - populate_from_params(agent, params) - agent.identifiers = update_identifiers(identifiers) - agent.affiliations = update_affiliations(affiliations) - - if agent.valid? - agent.save - else - error 400, agent.errors - end + agent = update_agent(agent, params) + + error 400, agent.errors unless agent.errors.empty? end halt 204 end @@ -71,69 +62,65 @@ class AgentsController < ApplicationController def update_identifiers(identifiers) Array(identifiers).map do |i| - identifier = LinkedData::Models::AgentIdentifier.find(i["notation"]).first + id = i["notation"] || (i["id"] || "").split('/').last + identifier = LinkedData::Models::AgentIdentifier.find(id).first + identifier = LinkedData::Models::AgentIdentifier.new unless identifier - if identifier - i.delete "id" - populate_from_params(identifier, i) - else - identifier = LinkedData::Models::AgentIdentifier.new(i) - end + i.delete "id" + populate_from_params(identifier, i) if identifier.valid? identifier.save else error 400, identifier.errors end - identifier end end def update_affiliations(affiliations) Array(affiliations).map do |aff| - affiliations = LinkedData::Models::Agent.find(aff["id"]) + affiliation = aff["id"] ? LinkedData::Models::Agent.find(RDF::URI.new(aff["id"])).first : nil - if affiliations - aff.delete "id" - populate_from_params(affiliations, aff) + if affiliation + affiliation.bring_remaining + affiliation = update_agent(affiliation, aff) else - affiliations = LinkedData::Models::Agent.new(aff) + affiliation = create_new_agent(aff["id"], aff) end - if affiliations.valid? - affiliations.save - else - error 400, affiliations.errors - end + error 400, affiliation.errors unless affiliation.errors.empty? - affiliations + affiliation end end - def create_agent - params ||= @params - acronym = params["id"] + def create_new_agent (id = @params['id'], params = @params) agent = nil - agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if acronym + agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if id if agent.nil? - identifiers = params.delete "identifiers" - affiliations = params.delete "affiliations" - params.delete "id" - agent = instance_from_params(LinkedData::Models::Agent, params) - agent.identifiers = update_identifiers(identifiers) - agent.affiliations = update_affiliations(affiliations) + agent = update_agent(LinkedData::Models::Agent.new, params) + error 400, agent.errors unless agent.errors.empty? + + return agent else error 400, "Agent exists, please use HTTP PATCH to update" end + end - if agent.valid? - agent.save - else - error 400, agent.errors - end - reply 201, agent + def update_agent(agent, params) + return agent unless agent + + identifiers = params.delete "identifiers" + affiliations = params.delete "affiliations" + params.delete "id" + populate_from_params(agent, params) + agent.identifiers = update_identifiers(identifiers) + agent.affiliations = update_affiliations(affiliations) + + agent.save if agent.valid? + return agent end end From 3375586abf332c57c1c95a7e990083854d142ea5 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 19 Jun 2023 01:10:23 +0200 Subject: [PATCH 39/70] add agent controller tests --- Gemfile.lock | 2 +- test/controllers/test_agents_controller.rb | 224 +++++++++++++++++++++ 2 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 test/controllers/test_agents_controller.rb diff --git a/Gemfile.lock b/Gemfile.lock index 0ebac83d..228a6e3e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 6f72110402537fffe295c4bfd6c1fd458f343942 + revision: f1edac81ea06d10ac91e98877b0283286fdc6535 branch: feature/add-persons-and-organization-models specs: ontologies_linked_data (0.0.1) diff --git a/test/controllers/test_agents_controller.rb b/test/controllers/test_agents_controller.rb new file mode 100644 index 00000000..d13e2b01 --- /dev/null +++ b/test/controllers/test_agents_controller.rb @@ -0,0 +1,224 @@ +require_relative '../test_case' +require "multi_json" + +class TestAgentsController < TestCase + + def setup + + @number_of_organizations = 6 + + + @test_agents = 8.times.map do |i| + type = i < @number_of_organizations ? 'organization' : 'person' + _agent_data(type: type) + end + @agents = [] + 2.times.map do + agents_tmp = [ _agent_data(type: 'organization'), _agent_data(type: 'organization'), _agent_data(type: 'person')] + agent = agents_tmp.last + agent[:affiliations] = [agents_tmp[0].stringify_keys, agents_tmp[1].stringify_keys] + _test_agent_creation(agent) + @agents = @agents + agents_tmp + end + end + + def teardown + # Delete groups + _delete_agents + end + + def test_all_agents + get '/agents' + assert last_response.ok? + + created_agents = MultiJson.load(last_response.body) + + @agents.each do |agent| + created_agent = created_agents.select{|x| x["name"].eql?(agent[:name])}.first + refute_nil created_agent + assert_equal agent[:name], created_agent["name"] + assert_equal agent[:identifiers].size, created_agent["identifiers"].size + assert_equal agent[:identifiers].map{|x| x[:notation]}.sort, created_agent["identifiers"].map{|x| x['notation']}.sort + assert_equal agent[:affiliations].size, created_agent["affiliations"].size + assert_equal agent[:affiliations].map{|x| x["name"]}.sort, created_agent["affiliations"].map{|x| x['name']}.sort + + end + end + + def test_single_agent + @agents.each do |agent| + agent_obj = _find_agent(agent['name']) + get "/agents/#{agent_obj.id.to_s.split('/').last}" + assert last_response.ok? + agent_found = MultiJson.load(last_response.body) + assert_equal agent_obj.id.to_s, agent_found["id"] + end + end + + def test_create_new_agent + + ## Create Agent of type affiliation with no parent affiliation + agent = @test_agents[0] + created_agent = _test_agent_creation(agent) + + ## Create Agent of type affiliation with an extent parent affiliation + + agent = @test_agents[1] + agent[:affiliations] = [created_agent] + + created_agent = _test_agent_creation(agent) + + ## Create Agent of type affiliation with an no extent parent affiliation + agent = @test_agents[3] + agent[:affiliations] = [created_agent, @test_agents[2].stringify_keys] + created_agent = _test_agent_creation(agent) + + ## Create Agent of type Person with an extent affiliations + + agent = @test_agents[6] + agent[:affiliations] = created_agent["affiliations"] + _test_agent_creation(agent) + + ## Create Agent of type Person with no extent affiliations + + agent = @test_agents[7] + agent[:affiliations] = [@test_agents[4].stringify_keys, @test_agents[5].stringify_keys] + _test_agent_creation(agent) + + @agents = @agents + @test_agents + end + + + def test_new_agent_no_valid + agents_tmp = [ _agent_data(type: 'organization'), _agent_data(type: 'person'), _agent_data(type: 'person')] + agent = agents_tmp.last + agent[:affiliations] = [agents_tmp[0].stringify_keys, agents_tmp[1].stringify_keys] + _test_agent_creation(agent) + end + + def test_update_patch_agent + + agents = [ _agent_data(type: 'organization'), _agent_data(type: 'organization'), _agent_data(type: 'person')] + agent = agents.last + agent[:affiliations] = [agents[0].stringify_keys, agents[1].stringify_keys] + agent = _test_agent_creation(agent) + @agents = @agents + agents + agent = LinkedData::Models::Agent.find(agent['id'].split('/').last).first + agent.bring_remaining + + + ## update identifiers + agent.identifiers.each{|i| i.bring_remaining} + new_identifiers = [] + ## update an existent identifier + new_identifiers[0] = { + id: agent.identifiers[0].id.to_s, + schemaAgency: 'TEST ' + agent.identifiers[0].notation + } + + new_identifiers[1] = { + id: agent.identifiers[1].id.to_s + } + + ## update affiliation + agent.affiliations.each{|aff| aff.bring_remaining} + new_affiliations = [] + ## update an existent affiliation + new_affiliations[0] = { + name: 'TEST new of ' + agent.affiliations[0].name, + id: agent.affiliations[0].id.to_s + } + ## create a new affiliation + new_affiliations[1] = _agent_data(type: 'organization') + new_affiliations[1][:name] = 'new affiliation' + + new_values = { + name: 'new name ', + identifiers: new_identifiers, + affiliations: new_affiliations + } + + patch "/agents/#{agent.id.split('/').last}", MultiJson.dump(new_values), "CONTENT_TYPE" => "application/json" + assert last_response.status == 204 + + get "/agents/#{agent.id.split('/').last}" + new_agent = MultiJson.load(last_response.body) + assert_equal 'new name ', new_agent["name"] + + assert_equal new_identifiers.size, new_agent["identifiers"].size + assert_equal new_identifiers[0][:schemaAgency], new_agent["identifiers"].select{|x| x["id"].eql?(agent.identifiers[0].id.to_s)}.first["schemaAgency"] + assert_equal agent.identifiers[1].schemaAgency, new_agent["identifiers"].select{|x| x["id"].eql?(agent.identifiers[1].id.to_s)}.first["schemaAgency"] + + assert_equal new_affiliations.size, new_agent["affiliations"].size + assert_equal new_affiliations[0][:name], new_agent["affiliations"].select{|x| x["id"].eql?(agent.affiliations[0].id.to_s)}.first["name"] + assert_nil new_agent["affiliations"].select{|x| x["id"].eql?(agent.affiliations[1].id.to_s)}.first + assert_equal new_affiliations[1][:name], new_agent["affiliations"].reject{|x| x["id"].eql?(agent.affiliations[0].id.to_s)}.first["name"] + end + + def test_delete_agent + agent = @agents.delete_at(0) + agent_obj = _find_agent(agent['name']) + id = agent_obj.id.to_s.split('/').last + delete "/agents/#{id}" + assert last_response.status == 204 + + get "/agents/#{id}" + assert last_response.status == 404 + end + + private + def _agent_data(type: 'organization') + schema_agencies = LinkedData::Models::AgentIdentifier::IDENTIFIER_SCHEMES.keys + users = LinkedData::Models::User.all + users = [LinkedData::Models::User.new(username: "tim", email: "tim@example.org", password: "password").save] if users.empty? + test_identifiers = 5.times.map { |i| { notation: rand.to_s[2..11], schemaAgency: schema_agencies.sample.to_s } } + user = users.sample.id.to_s + + i = rand.to_s[2..11] + return { + agentType: type, + name: "name #{i}", + homepage: "home page #{i}", + acronym: "acronym #{i}", + email: "email #{i}", + identifiers: test_identifiers.sample(2).map { |x| x.merge({ creator: user }) }, + affiliations: [], + creator: user + } + end + + def _find_agent(name) + LinkedData::Models::Agent.where(name: name).first + end + + def _delete_agents + @agents.each do |agent| + test_cat = _find_agent(agent[:name]) + next if test_cat.nil? + + test_cat.bring :identifiers + test_cat.identifiers.each { |i| i.delete } + test_cat.delete + end + end + + def _test_agent_creation(agent) + post "/agents", MultiJson.dump(agent), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 201 + created_agent = MultiJson.load(last_response.body) + assert created_agent["name"].eql?(agent[:name]) + + get "/agents/#{created_agent['id'].split('/').last}" + assert last_response.ok? + + created_agent = MultiJson.load(last_response.body) + assert_equal agent[:name], created_agent["name"] + assert_equal agent[:identifiers].size, created_agent["identifiers"].size + assert_equal agent[:identifiers].map { |x| x[:notation] }.sort, created_agent["identifiers"].map { |x| x['notation'] }.sort + + assert_equal agent[:affiliations].size, created_agent["affiliations"].size + assert_equal agent[:affiliations].map { |x| x["name"] }.sort, created_agent["affiliations"].map { |x| x['name'] }.sort + created_agent + end +end \ No newline at end of file From c785dfc5c7ba512220809c354f957529f97a209c Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 28 Jun 2023 00:35:28 +0200 Subject: [PATCH 40/70] don't update affiliations if only 'id' sent in params --- controllers/agents_controller.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index fa22586d..15cab96b 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -67,6 +67,9 @@ def update_identifiers(identifiers) identifier = LinkedData::Models::AgentIdentifier.new unless identifier i.delete "id" + + next identifier if i.keys.size.zero? + populate_from_params(identifier, i) if identifier.valid? @@ -82,8 +85,11 @@ def update_affiliations(affiliations) Array(affiliations).map do |aff| affiliation = aff["id"] ? LinkedData::Models::Agent.find(RDF::URI.new(aff["id"])).first : nil + affiliation.bring_remaining if affiliation + + next affiliation if aff.keys.size.eql?(1) && aff["id"] + if affiliation - affiliation.bring_remaining affiliation = update_agent(affiliation, aff) else affiliation = create_new_agent(aff["id"], aff) From 09a7eb7a7f092bb9aec270e1094cf7287e3d29a2 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 5 Jul 2023 15:25:26 +0200 Subject: [PATCH 41/70] bring identifier attributes when we update an agent --- controllers/agents_controller.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index 15cab96b..87572e99 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -62,9 +62,16 @@ class AgentsController < ApplicationController def update_identifiers(identifiers) Array(identifiers).map do |i| - id = i["notation"] || (i["id"] || "").split('/').last - identifier = LinkedData::Models::AgentIdentifier.find(id).first - identifier = LinkedData::Models::AgentIdentifier.new unless identifier + next nil if i.empty? + + id = i["id"] || LinkedData::Models::AgentIdentifier.generate_identifier(i['notation'], i['schemaAgency']) + identifier = LinkedData::Models::AgentIdentifier.find(RDF::URI.new(id)).first + + if identifier + identifier.bring_remaining + else + identifier = LinkedData::Models::AgentIdentifier.new + end i.delete "id" @@ -78,14 +85,17 @@ def update_identifiers(identifiers) error 400, identifier.errors end identifier - end + end.compact end def update_affiliations(affiliations) Array(affiliations).map do |aff| affiliation = aff["id"] ? LinkedData::Models::Agent.find(RDF::URI.new(aff["id"])).first : nil - affiliation.bring_remaining if affiliation + if affiliation + affiliation.bring_remaining + affiliation.identifiers.each{|i| i.bring_remaining} + end next affiliation if aff.keys.size.eql?(1) && aff["id"] From cf741667bab1d33973d67ac1d947e159710404b8 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 5 Jul 2023 15:26:13 +0200 Subject: [PATCH 42/70] update agent test to work with the new Agent validators --- Gemfile.lock | 12 ++++++------ test/controllers/test_agents_controller.rb | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 228a6e3e..50bf1730 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 39b7f2f2e9c398b981fb03feaa1eebc270877341 + revision: 6c1790433c6897dd398dce89ff724646e1c7e6af branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f1edac81ea06d10ac91e98877b0283286fdc6535 + revision: 596dc903e437e2b6af0893e8d68b9e4b4e640106 branch: feature/add-persons-and-organization-models specs: ontologies_linked_data (0.0.1) @@ -108,7 +108,7 @@ GEM airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) - bcrypt (3.1.18) + bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) @@ -171,7 +171,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.5.2) + googleauth (1.6.0) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.2.2) + newrelic_rpm (9.3.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -324,7 +324,7 @@ GEM systemu (2.6.5) temple (0.10.2) tilt (2.2.0) - timeout (0.3.2) + timeout (0.4.0) trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) diff --git a/test/controllers/test_agents_controller.rb b/test/controllers/test_agents_controller.rb index d13e2b01..ef0e5c47 100644 --- a/test/controllers/test_agents_controller.rb +++ b/test/controllers/test_agents_controller.rb @@ -93,7 +93,8 @@ def test_new_agent_no_valid agents_tmp = [ _agent_data(type: 'organization'), _agent_data(type: 'person'), _agent_data(type: 'person')] agent = agents_tmp.last agent[:affiliations] = [agents_tmp[0].stringify_keys, agents_tmp[1].stringify_keys] - _test_agent_creation(agent) + post "/agents", MultiJson.dump(agent), "CONTENT_TYPE" => "application/json" + assert last_response.status == 400 end def test_update_patch_agent @@ -180,7 +181,7 @@ def _agent_data(type: 'organization') name: "name #{i}", homepage: "home page #{i}", acronym: "acronym #{i}", - email: "email #{i}", + email: "email_#{i}@test.com", identifiers: test_identifiers.sample(2).map { |x| x.merge({ creator: user }) }, affiliations: [], creator: user From 070b1fb3cd1b50edc11dbbb3c8b8d9fd50e089cb Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sun, 30 Jul 2023 02:13:29 +0200 Subject: [PATCH 43/70] bring the agent attributes on display all of the submissions endpoints --- Gemfile | 5 +-- Gemfile.lock | 31 ++++++++++--------- controllers/ontologies_controller.rb | 5 +-- .../ontology_submissions_controller.rb | 4 +-- helpers/submission_helper.rb | 30 +++++++++++------- 5 files changed, 40 insertions(+), 35 deletions(-) diff --git a/Gemfile b/Gemfile index 8326c2dd..418c74ae 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'activesupport', '~> 3.0' +gem 'activesupport', '~> 3.1' # see https://github.com/ncbo/ontologies_api/issues/69 gem 'bigdecimal', '1.4.2' gem 'faraday', '~> 1.9' @@ -19,7 +19,7 @@ gem 'request_store' gem 'ffi' gem 'rack-accept', '~> 0.4' gem 'rack-attack', '~> 6.6.1', require: 'rack/attack' -gem 'rack-cache', '~> 1.0' +gem 'rack-cache', '~> 1.13.0' gem 'rack-cors', require: 'rack/cors' # GitHub dependency can be removed when https://github.com/niko/rack-post-body-to-params/pull/6 is merged and released gem 'rack-post-body-to-params', github: 'palexander/rack-post-body-to-params', branch: 'multipart_support' @@ -29,6 +29,7 @@ gem 'redis-rack-cache', '~> 2.0' # Data access (caching) gem 'redis', '~> 4.8.1' gem 'redis-activesupport' +gem 'redis-store', '1.9.1' # Monitoring gem 'cube-ruby', require: 'cube' diff --git a/Gemfile.lock b/Gemfile.lock index 50bf1730..f643627b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: d0ac992c88bd417f2f2137ba62934c3c41b6db7c + revision: 83e835de368bc9f19da800a477982e0ad770900d branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 6c1790433c6897dd398dce89ff724646e1c7e6af + revision: 1d78bde5a711d05475da0459308c7db074af5e21 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 596dc903e437e2b6af0893e8d68b9e4b4e640106 + revision: 04dc11434af0d3eae9861e378f84be9bec4a0d9e branch: feature/add-persons-and-organization-models specs: ontologies_linked_data (0.0.1) @@ -162,7 +162,7 @@ GEM ffi (~> 1.0) google-apis-analytics_v3 (0.13.0) google-apis-core (>= 0.11.0, < 2.a) - google-apis-core (0.11.0) + google-apis-core (0.11.1) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -171,7 +171,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.6.0) + googleauth (1.7.0) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -215,7 +215,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.3.6) + net-imap (0.3.7) date net-protocol net-pop (0.1.2) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.3.0) + newrelic_rpm (9.3.1) oj (2.18.5) omni_logger (0.1.4) logger @@ -239,7 +239,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.1) + public_suffix (5.0.3) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -268,8 +268,8 @@ GEM redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.2) - redis (>= 4, < 6) + redis-store (1.9.1) + redis (>= 4, < 5) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -282,7 +282,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.2.5) + rexml (3.2.6) rsolr (2.5.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) @@ -318,7 +318,7 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.4) + sshkit (1.21.5) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) @@ -347,7 +347,7 @@ PLATFORMS x86_64-linux DEPENDENCIES - activesupport (~> 3.0) + activesupport (~> 3.1) bcrypt_pbkdf (>= 1.0, < 2.0) bigdecimal (= 1.4.2) capistrano (~> 3) @@ -375,7 +375,7 @@ DEPENDENCIES rack rack-accept (~> 0.4) rack-attack (~> 6.6.1) - rack-cache (~> 1.0) + rack-cache (~> 1.13.0) rack-cors rack-mini-profiler rack-post-body-to-params! @@ -386,6 +386,7 @@ DEPENDENCIES redis (~> 4.8.1) redis-activesupport redis-rack-cache (~> 2.0) + redis-store (= 1.9.1) request_store shotgun! simplecov @@ -398,4 +399,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.14 + 2.3.23 diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index 1a8f2ebf..99c0ce68 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -44,10 +44,7 @@ class OntologiesController < ApplicationController if includes_param.first == :all # Bring what we need to display all attr of the submission latest.bring_remaining - latest.bring({:contact=>[:name, :email], - :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, - :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) + latest.bring(*submission_attributes_all) else includes = OntologySubmission.goo_attrs_to_load(includes_param) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 96411a8d..7de7a3dc 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -31,9 +31,7 @@ class OntologySubmissionsController < ApplicationController check_access(ont) if includes_param.first == :all # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) - ont.bring(submissions: [:released, :creationDate, :status, :submissionId, - {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus], :metrics =>[:classes, :individuals, :properties]) + ont.bring(submission_attributes_all) ont.submissions.each do |sub| sub.bring_remaining diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 534ddbd8..c1ee5dd3 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -4,20 +4,28 @@ module Sinatra module Helpers module SubmissionHelper + def submission_attributes_all + out = [LinkedData::Models::OntologySubmission.embed_values_hash] + out << {:contact=>[:name, :email]} + out << {:ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, + :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType]} + + out + end + def retrieve_submissions(options) status = (options[:status] || "RDF").to_s.upcase status = "RDF" if status.eql?("READY") any = status.eql?("ANY") include_views = options[:also_include_views] || false - includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) + includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) includes << :submissionStatus unless includes.include?(:submissionStatus) - submissions_query = LinkedData::Models::OntologySubmission if any submissions_query = submissions_query.where else - submissions_query = submissions_query.where({submissionStatus: [ code: status]}) + submissions_query = submissions_query.where({ submissionStatus: [code: status] }) end submissions_query = apply_submission_filters(submissions_query) @@ -26,18 +34,18 @@ def retrieve_submissions(options) # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], - :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, + includes = [:submissionId, { :contact => [:name, :email], + :ontology => [:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], + :submissionStatus => [:code], :hasOntologyLanguage => [:acronym], :metrics => [:classes, :individuals, :properties] }, :submissionStatus] else - if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} + if includes.find { |v| v.is_a?(Hash) && v.keys.include?(:ontology) } + includes << { :ontology => [:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain, :notes, :reviews, :projects, :acl, :viewOf] } end - if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} - includes << {:contact=>[:name, :email]} + if includes.find { |v| v.is_a?(Hash) && v.keys.include?(:contact) } + includes << { :contact => [:name, :email] } end end From f0da26ee1bae894e686e908328a241d0aa019099 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 31 Jul 2023 06:20:07 +0200 Subject: [PATCH 44/70] handle exception for class attribute but aren't in populate_from_params --- helpers/application_helper.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 84c26497..5d6d1b0a 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -51,6 +51,10 @@ def populate_from_params(obj, params) value = is_arr ? value : [value] new_value = [] value.each do |cls| + if uri_as_needed(cls["ontology"]).nil? + new_value << cls + next + end sub = LinkedData::Models::Ontology.find(uri_as_needed(cls["ontology"])).first.latest_submission new_value << LinkedData::Models::Class.find(cls["class"]).in(sub).first end From f37895034c44e94a764212871fe31b6f6e9141cf Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 1 Aug 2023 03:26:39 +0200 Subject: [PATCH 45/70] update Gemfile to use development branch of OLD --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 418c74ae..127ad02f 100644 --- a/Gemfile +++ b/Gemfile @@ -49,7 +49,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/add-persons-and-organization-models' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index f643627b..dc312b40 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 04dc11434af0d3eae9861e378f84be9bec4a0d9e - branch: feature/add-persons-and-organization-models + revision: ec1e02def3ad2480e08322f65c564ffb731bda6a + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -226,7 +226,7 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-smtp (0.3.3) net-protocol - net-ssh (7.1.0) + net-ssh (7.2.0) netrc (0.11.0) newrelic_rpm (9.3.1) oj (2.18.5) From 3d05980f8ddf1a6909feb2094e02c981d3751e34 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 3 Aug 2023 19:35:41 +0200 Subject: [PATCH 46/70] refactor user controller to extract reset password helpers --- controllers/users_controller.rb | 24 +++++++----------------- helpers/users_helper.rb | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index 00b6e732..4f89f785 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -20,17 +20,13 @@ class UsersController < ApplicationController post "/create_reset_password_token" do email = params["email"] username = params["username"] - user = LinkedData::Models::User.where(email: email, username: username).include(LinkedData::Models::User.attributes).first - error 404, "User not found" unless user - reset_token = token(36) - user.resetToken = reset_token + user = send_reset_token(email, username) + if user.valid? - user.save(override_security: true) - LinkedData::Utils::Notifications.reset_password(user, reset_token) + halt 204 else error 422, user.errors end - halt 204 end ## @@ -42,11 +38,11 @@ class UsersController < ApplicationController email = params["email"] || "" username = params["username"] || "" token = params["token"] || "" + params["display"] = User.attributes.join(",") # used to serialize everything via the serializer - user = LinkedData::Models::User.where(email: email, username: username).include(User.goo_attrs_to_load(includes_param)).first - error 404, "User not found" unless user - if token.eql?(user.resetToken) - user.show_apikey = true + + user, token_accepted = reset_password(email, username, token) + if token_accepted reply user else error 403, "Password reset not authorized with this token" @@ -98,12 +94,6 @@ class UsersController < ApplicationController private - def token(len) - chars = ("a".."z").to_a + ("A".."Z").to_a + ("1".."9").to_a - token = "" - 1.upto(len) { |i| token << chars[rand(chars.size-1)] } - token - end def create_user params ||= @params diff --git a/helpers/users_helper.rb b/helpers/users_helper.rb index 5d4266c1..00cd7aa0 100644 --- a/helpers/users_helper.rb +++ b/helpers/users_helper.rb @@ -17,6 +17,37 @@ def filter_for_user_onts(obj) obj end + + def send_reset_token(email, username) + user = LinkedData::Models::User.where(email: email, username: username).include(LinkedData::Models::User.attributes).first + error 404, "User not found" unless user + reset_token = token(36) + user.resetToken = reset_token + + return user if user.valid? + + user.save(override_security: true) + LinkedData::Utils::Notifications.reset_password(user, reset_token) + user + end + + def token(len) + chars = ("a".."z").to_a + ("A".."Z").to_a + ("1".."9").to_a + token = "" + 1.upto(len) { |i| token << chars[rand(chars.size-1)] } + token + end + + def reset_password(email, username, token) + user = LinkedData::Models::User.where(email: email, username: username).include(User.goo_attrs_to_load(includes_param)).first + + error 404, "User not found" unless user + + user.show_apikey = true + + [user, token.eql?(user.resetToken)] + end + end end end From ae6bdd5fb512cb06ddd7dd0b5c68da8606330d0f Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 3 Aug 2023 19:36:41 +0200 Subject: [PATCH 47/70] remove the send notification on user creation, now handled by user.save --- controllers/users_controller.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index 4f89f785..716ebff1 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -101,14 +101,7 @@ def create_user error 409, "User with username `#{params["username"]}` already exists" unless user.nil? user = instance_from_params(User, params) if user.valid? - user.save - # Send an email to the administrator to warn him about the newly created user - begin - if !LinkedData.settings.admin_emails.nil? && !LinkedData.settings.admin_emails.empty? - LinkedData::Utils::Notifications.new_user(user) - end - rescue Exception => e - end + user.save(send_notifications: false) else error 422, user.errors end From 2cdd4ffcca18a3c0d44a19b6bfb4744f2c40b15a Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 3 Aug 2023 20:02:51 +0200 Subject: [PATCH 48/70] add access token authentication --- Gemfile | 2 +- Gemfile.lock | 9 +++++---- controllers/users_controller.rb | 15 +++++++++------ helpers/users_helper.rb | 18 ++++++++++++++++++ 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index 71cdf420..47469ba0 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/add-multi-provider-authentification' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 1d512537..29a93d56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 4ac9873f3d016259196a17cfc9c6ab8149468ec9 - branch: development + revision: 29c77d69da0c0566a450bc5f81ae6ce243e84b25 + branch: feature/add-multi-provider-authentification specs: ontologies_linked_data (0.0.1) activesupport @@ -103,7 +103,7 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.4) + addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) @@ -249,7 +249,7 @@ GEM rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) - rack-mini-profiler (3.1.0) + rack-mini-profiler (3.1.1) rack (>= 1.2.0) rack-protection (1.5.5) rack @@ -343,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index 716ebff1..01e4353a 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -1,14 +1,17 @@ class UsersController < ApplicationController namespace "/users" do post "/authenticate" do - user_id = params["user"] - user_password = params["password"] + # Modify params to show all user attributes params["display"] = User.attributes.join(",") - user = User.find(user_id).include(User.goo_attrs_to_load(includes_param) + [:passwordHash]).first - authenticated = user.authenticate(user_password) unless user.nil? - error 401, "Username/password combination invalid" unless authenticated - user.show_apikey = true + + if params["access_token"] + user = oauth_authenticate(params) + else + user = login_password_authenticate(params) + end + user.bring_remaining + user.show_apikey = true unless user.nil? reply user end diff --git a/helpers/users_helper.rb b/helpers/users_helper.rb index 00cd7aa0..16a2d0e6 100644 --- a/helpers/users_helper.rb +++ b/helpers/users_helper.rb @@ -48,6 +48,24 @@ def reset_password(email, username, token) [user, token.eql?(user.resetToken)] end + def oauth_authenticate(params) + access_token = params["access_token"] + provider = params["token_provider"] + user = LinkedData::Models::User.oauth_authenticate(access_token, provider) + error 401, "Access token invalid" unless user.nil? + user + end + + def login_password_authenticate(params) + user_id = params["user"] + user_password = params["password"] + user = User.find(user_id).include(User.goo_attrs_to_load(includes_param) + [:passwordHash]).first + authenticated = false + authenticated = user.authenticate(user_password) unless user.nil? + error 401, "Username/password combination invalid" unless authenticated + + user + end end end end From 54db3c0f0b889efecea06e938dfe6766cc2a0e40 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Fri, 4 Aug 2023 00:56:36 +0200 Subject: [PATCH 49/70] fix test after enforcing the uniqueness of user emails --- test/controllers/test_ontologies_controller.rb | 4 ++-- test/controllers/test_ontology_submissions_controller.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4713b699..dc79359b 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -217,13 +217,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test1@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test2@example.org", password: "12345" }) blocked_user.save diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index ee8576bd..9ee81257 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,7 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}], + contact: [{name: "test_name", email: "test3@example.org"}], URI: 'https://test.com/test', status: 'production', description: 'ontology description' @@ -159,13 +159,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test4@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test5@example.org", password: "12345" }) blocked_user.save From 3d79e402a6a4d239ee865c410c087c4cfa820b80 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Fri, 4 Aug 2023 01:42:34 +0200 Subject: [PATCH 50/70] fix search test --- .dockerignore | 1 + Gemfile.lock | 38 ++++++++--------- docker-compose.yml | 16 +++++++- helpers/search_helper.rb | 1 + test/controllers/test_search_controller.rb | 27 ++++++++---- .../thesaurusINRAE_nouv_structure.rdf | 2 +- test/middleware/test_rack_attack.rb | 6 +-- .../configsets/term_search/conf/schema.xml | 41 ++++++++++++++++--- 8 files changed, 94 insertions(+), 38 deletions(-) diff --git a/.dockerignore b/.dockerignore index cf76ed57..3b15d33c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,4 @@ tmp/* # Editor temp files *.swp *.swo +test/solr diff --git a/Gemfile.lock b/Gemfile.lock index 29a93d56..5b1acbd9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: d0ac992c88bd417f2f2137ba62934c3c41b6db7c + revision: 83e835de368bc9f19da800a477982e0ad770900d branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: c3456c45c12ed92d4a3ae43cac7c1d4cdbf290b6 + revision: 1d78bde5a711d05475da0459308c7db074af5e21 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 29c77d69da0c0566a450bc5f81ae6ce243e84b25 + revision: f398b405f588d1161bc4e4893f5741360eb576f2 branch: feature/add-multi-provider-authentification specs: ontologies_linked_data (0.0.1) @@ -108,11 +108,11 @@ GEM airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) - bcrypt (3.1.18) + bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.2) + capistrano (3.17.3) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -162,7 +162,7 @@ GEM ffi (~> 1.0) google-apis-analytics_v3 (0.13.0) google-apis-core (>= 0.11.0, < 2.a) - google-apis-core (0.11.0) + google-apis-core (0.11.1) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -171,7 +171,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.5.2) + googleauth (1.7.0) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -191,9 +191,9 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.6.3) - jwt (2.7.0) + jwt (2.7.1) kgio (2.11.4) - libxml-ruby (4.1.0) + libxml-ruby (4.1.1) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) @@ -215,7 +215,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.3.4) + net-imap (0.3.7) date net-protocol net-pop (0.1.2) @@ -226,9 +226,9 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-smtp (0.3.3) net-protocol - net-ssh (7.1.0) + net-ssh (7.2.0) netrc (0.11.0) - newrelic_rpm (9.2.2) + newrelic_rpm (9.3.1) oj (2.18.5) omni_logger (0.1.4) logger @@ -239,13 +239,13 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.1) + public_suffix (5.0.3) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.6.1) rack (>= 1.0, < 3) - rack-cache (1.13.0) + rack-cache (1.14.0) rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) @@ -282,7 +282,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.2.5) + rexml (3.2.6) rsolr (2.5.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) @@ -318,13 +318,13 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.4) + sshkit (1.21.5) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.10.0) - tilt (2.1.0) - timeout (0.3.2) + temple (0.10.2) + tilt (2.2.0) + timeout (0.4.0) trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) diff --git a/docker-compose.yml b/docker-compose.yml index de084081..5cb64963 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,10 +75,14 @@ services: redis-ut: image: redis + ports: + - 6379:6379 4store-ut: image: bde2020/4store #volume: fourstore:/var/lib/4store + ports: + - 9000:9000 command: > bash -c "4s-backend-setup --segments 4 ontoportal_kb && 4s-backend ontoportal_kb @@ -88,10 +92,20 @@ services: solr-ut: - image: ontoportal/solr-ut:0.1 + image: solr:8 + volumes: + - ./test/solr/configsets:/configsets:ro + ports: + - "8983:8983" + command: > + bash -c "precreate-core term_search_core1 /configsets/term_search + && precreate-core prop_search_core1 /configsets/property_search + && solr-foreground" mgrep-ut: image: ontoportal/mgrep-ncbo:0.1 + ports: + - "55556:55555" agraph-ut: image: franzinc/agraph:v7.3.0 diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 10de14c0..5d37d884 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -345,6 +345,7 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) doc[:submission] = old_class.submission doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = LinkedData::Models::Class.read_only(doc) + instance.prefLabel = instance.prefLabel.first if instance.prefLabel.is_a?(Array) classes_hash[ont_uri_class_uri] = instance end diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 44c67c7e..21a3dd18 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -85,7 +85,7 @@ def test_search_ontology_filter assert last_response.ok? results = MultiJson.load(last_response.body) doc = results["collection"][0] - assert_equal "cell line", doc["prefLabel"] + assert_equal "cell line", doc["prefLabel"].first assert doc["links"]["ontology"].include? acronym results["collection"].each do |doc| acr = doc["links"]["ontology"].split('/')[-1] @@ -103,7 +103,8 @@ def test_search_other_filters get "search?q=data&require_definitions=true" assert last_response.ok? results = MultiJson.load(last_response.body) - assert_equal 26, results["collection"].length + assert results["collection"].all? {|doc| !doc["definition"].nil? && doc.values.flatten.join(" ").include?("data") } + #assert_equal 26, results["collection"].length get "search?q=data&require_definitions=false" assert last_response.ok? @@ -115,10 +116,14 @@ def test_search_other_filters get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}" results = MultiJson.load(last_response.body) - assert_equal 22, results["collection"].length + + assert results["collection"].all? { |x| !x["obsolete"] } + count = results["collection"].length + get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}&also_search_obsolete=false" results = MultiJson.load(last_response.body) - assert_equal 22, results["collection"].length + assert_equal count, results["collection"].length + get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}&also_search_obsolete=true" results = MultiJson.load(last_response.body) assert_equal 29, results["collection"].length @@ -134,8 +139,14 @@ def test_search_other_filters # testing cui and semantic_types flags get "search?q=Funding%20Resource&ontologies=#{acronym}&include=prefLabel,synonym,definition,notation,cui,semanticType" results = MultiJson.load(last_response.body) - assert_equal 35, results["collection"].length - assert_equal "Funding Resource", results["collection"][0]["prefLabel"] + #assert_equal 35, results["collection"].length + assert results["collection"].all? do |r| + ["prefLabel", "synonym", "definition", "notation", "cui", "semanticType"].map {|x| r[x]} + .flatten + .join(' ') + .include?("Funding Resource") + end + assert_equal "Funding Resource", results["collection"][0]["prefLabel"].first assert_equal "T028", results["collection"][0]["semanticType"][0] assert_equal "X123456", results["collection"][0]["cui"][0] @@ -190,7 +201,7 @@ def test_search_provisional_class assert_equal 10, results["collection"].length provisional = results["collection"].select {|res| assert_equal ontology_type, res["ontologyType"]; res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_root.label, provisional[0]["prefLabel"] + assert_equal @@test_pc_root.label, provisional[0]["prefLabel"].first # subtree root with provisional class test get "search?ontology=#{acronym}&subtree_root_id=#{CGI::escape(@@cls_uri.to_s)}&also_search_provisional=true" @@ -199,7 +210,7 @@ def test_search_provisional_class provisional = results["collection"].select {|res| res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_child.label, provisional[0]["prefLabel"] + assert_equal @@test_pc_child.label, provisional[0]["prefLabel"].first end end diff --git a/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf b/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf index 8353d82f..ca303834 100644 --- a/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf +++ b/test/data/ontology_files/thesaurusINRAE_nouv_structure.rdf @@ -30,7 +30,7 @@ 1331561625299 - aktivite + aktivite 2012-03-12T22:13:45Z 2017-09-22T14:09:06Z diff --git a/test/middleware/test_rack_attack.rb b/test/middleware/test_rack_attack.rb index 43143080..0b10c9e1 100644 --- a/test/middleware/test_rack_attack.rb +++ b/test/middleware/test_rack_attack.rb @@ -18,14 +18,14 @@ def self.before_suite LinkedData::OntologiesAPI.settings.req_per_second_per_ip = 1 LinkedData::OntologiesAPI.settings.safe_ips = Set.new(["1.2.3.4", "1.2.3.5"]) - @@user = LinkedData::Models::User.new({username: "user", password: "test_password", email: "test_email@example.org"}) + @@user = LinkedData::Models::User.new({username: "user", password: "test_password", email: "test_email1@example.org"}) @@user.save - @@bp_user = LinkedData::Models::User.new({username: "ncbobioportal", password: "test_password", email: "test_email@example.org"}) + @@bp_user = LinkedData::Models::User.new({username: "ncbobioportal", password: "test_password", email: "test_email2@example.org"}) @@bp_user.save admin_role = LinkedData::Models::Users::Role.find("ADMINISTRATOR").first - @@admin = LinkedData::Models::User.new({username: "admin", password: "test_password", email: "test_email@example.org", role: [admin_role]}) + @@admin = LinkedData::Models::User.new({username: "admin", password: "test_password", email: "test_email3@example.org", role: [admin_role]}) @@admin.save # Redirect output or we get a bunch of noise from Rack (gets reset in the after_suite method). diff --git a/test/solr/configsets/term_search/conf/schema.xml b/test/solr/configsets/term_search/conf/schema.xml index 6b18a2a1..fa95e127 100644 --- a/test/solr/configsets/term_search/conf/schema.xml +++ b/test/solr/configsets/term_search/conf/schema.xml @@ -128,11 +128,20 @@ - - - - - + + + + + + + + + + + + + + @@ -140,9 +149,18 @@ + + + + + + + - + + + @@ -251,6 +269,17 @@ + + + + + + + + + + + From ac6d356ec49843079d02f75779f7591ebc417029 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Fri, 4 Aug 2023 10:45:49 +0200 Subject: [PATCH 51/70] add oauth_authentication test --- Gemfile | 1 + Gemfile.lock | 12 ++++++-- config/environments/test.rb | 18 ++++++++++++ helpers/users_helper.rb | 2 +- test/controllers/test_users_controller.rb | 36 +++++++++++++++++++++++ test/test_case.rb | 2 ++ 6 files changed, 68 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 47469ba0..dba5dbf7 100644 --- a/Gemfile +++ b/Gemfile @@ -72,4 +72,5 @@ group :test do gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io + gem 'webmock' end diff --git a/Gemfile.lock b/Gemfile.lock index 5b1acbd9..21e9b895 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f398b405f588d1161bc4e4893f5741360eb576f2 + revision: e4b3a6d9bf575c1420924d4dbe1490248040aff7 branch: feature/add-multi-provider-authentification specs: ontologies_linked_data (0.0.1) @@ -126,6 +126,8 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.2) + crack (0.4.5) + rexml cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -181,6 +183,7 @@ GEM haml (5.2.2) temple (>= 0.8.0) tilt + hashdiff (1.0.1) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) @@ -245,7 +248,7 @@ GEM rack (>= 0.4) rack-attack (6.6.1) rack (>= 1.0, < 3) - rack-cache (1.14.0) + rack-cache (1.13.0) rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) @@ -340,6 +343,10 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.18.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) webrick (1.8.1) PLATFORMS @@ -396,6 +403,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer + webmock BUNDLED WITH 2.3.14 diff --git a/config/environments/test.rb b/config/environments/test.rb index 0f421dec..16bf407a 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -55,6 +55,24 @@ "apikey" => "1cfae05f-9e67-486f-820b-b393dec5764b" } } + config.oauth_providers = { + github: { + check: :access_token, + link: 'https://api.github.com/user' + }, + keycloak: { + check: :jwt_token, + cert: 'KEYCLOAK_SECRET_KEY' + }, + orcid: { + check: :access_token, + link: 'https://pub.orcid.org/v3.0/me' + }, + google: { + check: :access_token, + link: 'https://www.googleapis.com/oauth2/v3/userinfo' + } + } end Annotator.config do |config| diff --git a/helpers/users_helper.rb b/helpers/users_helper.rb index 16a2d0e6..e2c69e60 100644 --- a/helpers/users_helper.rb +++ b/helpers/users_helper.rb @@ -52,7 +52,7 @@ def oauth_authenticate(params) access_token = params["access_token"] provider = params["token_provider"] user = LinkedData::Models::User.oauth_authenticate(access_token, provider) - error 401, "Access token invalid" unless user.nil? + error 401, "Access token invalid"if user.nil? user end diff --git a/test/controllers/test_users_controller.rb b/test/controllers/test_users_controller.rb index 337da52e..3710b503 100644 --- a/test/controllers/test_users_controller.rb +++ b/test/controllers/test_users_controller.rb @@ -100,4 +100,40 @@ def test_authentication assert user["username"].eql?(@@usernames.first) end + def test_oauth_authentication + fake_responses = { + github: { + id: 123456789, + login: 'github_user', + email: 'github_user@example.com', + name: 'GitHub User', + avatar_url: 'https://avatars.githubusercontent.com/u/123456789' + }, + google: { + sub: 'google_user_id', + email: 'google_user@example.com', + name: 'Google User', + given_name: 'Google', + family_name: 'User', + picture: 'https://lh3.googleusercontent.com/a-/user-profile-image-url' + }, + orcid: { + orcid: '0000-0002-1825-0097', + email: 'orcid_user@example.com', + name: { + "family-name": 'ORCID', + "given-names": 'User' + } + } + } + + fake_responses.each do |provider, data| + WebMock.stub_request(:get, LinkedData::Models::User.oauth_providers[provider][:link]) + .to_return(status: 200, body: data.to_json, headers: { 'Content-Type' => 'application/json' }) + post "/users/authenticate", {access_token:'jkooko', token_provider: provider.to_s} + assert last_response.ok? + user = MultiJson.load(last_response.body) + assert data[:email], user["email"] + end + end end diff --git a/test/test_case.rb b/test/test_case.rb index 7d3d0716..be162d5e 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -21,7 +21,9 @@ require_relative 'test_log_file' require_relative '../app' require 'minitest/unit' +require 'webmock/minitest' MiniTest::Unit.autorun +WebMock.allow_net_connect! require 'rack/test' require 'multi_json' require 'oj' From fce1674b530c92905766c28c0f28145feba917c4 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sun, 6 Aug 2023 02:58:27 +0200 Subject: [PATCH 52/70] bring the correct attributes when the oauth_authenticate is used --- controllers/users_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index 01e4353a..09a1835b 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -7,10 +7,10 @@ class UsersController < ApplicationController if params["access_token"] user = oauth_authenticate(params) + user.bring(*User.goo_attrs_to_load(includes_param)) else user = login_password_authenticate(params) end - user.bring_remaining user.show_apikey = true unless user.nil? reply user end From ff7650ee99a3e4dadc807fc9b8dfc97ee87c2d54 Mon Sep 17 00:00:00 2001 From: HADDAD Zineddine Date: Tue, 5 Sep 2023 09:14:58 +0200 Subject: [PATCH 53/70] Feature: Add support of multilingual search (#40) * update get_term_search_query to support multilanguages search * rename var * fix search lang suffix to use underscore not @ * add multilangual search test --------- Co-authored-by: Syphax Bouazzouni --- Gemfile.lock | 2 +- helpers/search_helper.rb | 15 +++++--- test/controllers/test_search_controller.rb | 44 ++++++++++++++++++++++ test/data/ontology_files/BRO_v3.2.owl | 3 ++ test/solr/docker-compose.yml | 13 +++++++ test/solr/generate_ncbo_configsets.sh | 35 +++++++++-------- 6 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 test/solr/docker-compose.yml diff --git a/Gemfile.lock b/Gemfile.lock index e4bda0b3..44537959 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -248,7 +248,7 @@ GEM rack (>= 0.4) rack-attack (6.6.1) rack (>= 1.0, < 3) - rack-cache (1.13.0) + rack-cache (1.14.0) rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 5d37d884..071000d9 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -82,6 +82,9 @@ def get_term_search_query(text, params={}) end end + lang = params["lang"] || params["language"] + lang_suffix = lang && !lang.eql?("all") ? "_#{lang}" : "" + query = "" params["defType"] = "edismax" params["stopwords"] = "true" @@ -98,15 +101,15 @@ def get_term_search_query(text, params={}) if params[EXACT_MATCH_PARAM] == "true" query = "\"#{solr_escape(text)}\"" - params["qf"] = "resource_id^20 prefLabelExact^10 synonymExact #{QUERYLESS_FIELDS_STR}" - params["hl.fl"] = "resource_id prefLabelExact synonymExact #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^20 prefLabelExact#{lang_suffix }^10 synonymExact#{lang_suffix } #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix } synonymExact#{lang_suffix } #{QUERYLESS_FIELDS_STR}" elsif params[SUGGEST_PARAM] == "true" || text[-1] == '*' text.gsub!(/\*+$/, '') query = "\"#{solr_escape(text)}\"" params["qt"] = "/suggest_ncbo" - params["qf"] = "prefLabelExact^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["qf"] = "prefLabelExact#{lang_suffix }^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" params["pf"] = "prefLabelSuggest^50" - params["hl.fl"] = "prefLabelExact prefLabelSuggestEdge synonymSuggestEdge prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "prefLabelExact#{lang_suffix } prefLabelSuggestEdge synonymSuggestEdge prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" else if text.strip.empty? query = '*' @@ -114,9 +117,9 @@ def get_term_search_query(text, params={}) query = solr_escape(text) end - params["qf"] = "resource_id^100 prefLabelExact^90 prefLabel^70 synonymExact^50 synonym^10 #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^100 prefLabelExact#{lang_suffix }^90 prefLabel#{lang_suffix }^70 synonymExact#{lang_suffix }^50 synonym#{lang_suffix }^10 #{QUERYLESS_FIELDS_STR}" params["qf"] << " property" if params[INCLUDE_PROPERTIES_PARAM] == "true" - params["hl.fl"] = "resource_id prefLabelExact prefLabel synonymExact synonym #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix } prefLabel#{lang_suffix } synonymExact#{lang_suffix } synonym#{lang_suffix } #{QUERYLESS_FIELDS_STR}" params["hl.fl"] = "#{params["hl.fl"]} property" if params[INCLUDE_PROPERTIES_PARAM] == "true" end diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 21a3dd18..74be75d2 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -213,4 +213,48 @@ def test_search_provisional_class assert_equal @@test_pc_child.label, provisional[0]["prefLabel"].first end + def test_multilingual_search + get "/search?q=Activity&ontologies=BROSEARCHTEST-0" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + + doc = res["collection"].select{|doc| doc["@id"].to_s.eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + refute_nil doc + + #res = LinkedData::Models::Class.search("prefLabel_none:Activity", {:fq => "submissionAcronym:BROSEARCHTEST-0", :start => 0, :rows => 80}, :main) + #refute_equal 0, res["response"]["numFound"] + #refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activity&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + end + end diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index d64075cc..b2aeccf5 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -631,6 +631,9 @@ Activity + Activity + ActivityEnglish + Activité Activity of interest that may be related to a BRO:Resource. activities diff --git a/test/solr/docker-compose.yml b/test/solr/docker-compose.yml new file mode 100644 index 00000000..3ddae69c --- /dev/null +++ b/test/solr/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.8' + +services: + op_solr: + image: solr:8.8 + volumes: + - ./solr_configsets:/configsets:ro + ports: + - "8983:8983" + command: > + bash -c "precreate-core term_search_core1 /configsets/term_search + && precreate-core prop_search_core1 /configsets/property_search + && solr-foreground" diff --git a/test/solr/generate_ncbo_configsets.sh b/test/solr/generate_ncbo_configsets.sh index 893f7f3a..7b4281f7 100755 --- a/test/solr/generate_ncbo_configsets.sh +++ b/test/solr/generate_ncbo_configsets.sh @@ -2,18 +2,23 @@ # generates solr configsets by merging _default configset with config files in config/solr # _default is copied from sorl distribuion solr-8.10.1/server/solr/configsets/_default/ -pushd solr/configsets -ld_config='../../../../ontologies_linked_data/config/solr/' -#ld_config='../../../../config/solr/' -ls -l $ld_config -pwd -[ -d property_search ] && rm -Rf property_search -[ -d term_search ] && rm -Rf property_search -[ -d $ld_config/property_search ] || echo "cant find ontologies_linked_data project" -mkdir -p property_search/conf -mkdir -p term_search/conf -cp -a _default/conf/* property_search/conf/ -cp -a _default/conf/* term_search/conf/ -cp -a $ld_config/property_search/* property_search/conf -cp -a $ld_config/term_search/* term_search/conf -popd +#cd solr/configsets +ld_config='config/solr' +configsets='test/solr/configsets' +[ -d ${configsets}/property_search ] && rm -Rf ${configsets}/property_search +[ -d ${configsets}/term_search ] && rm -Rf ${configsets}/term_search +if [[ ! -d ${ld_config}/property_search ]]; then + echo 'cant find ld solr config sets' + exit 1 +fi +if [[ ! -d ${configsets}/_default/conf ]]; then + echo 'cant find default solr configset' + exit 1 +fi +mkdir -p ${configsets}/property_search/conf +mkdir -p ${configsets}/term_search/conf +cp -a ${configsets}/_default/conf/* ${configsets}/property_search/conf/ +cp -a ${configsets}/_default/conf/* ${configsets}/term_search/conf/ +cp -a $ld_config/property_search/* ${configsets}/property_search/conf +cp -a $ld_config/term_search/* ${configsets}/term_search/conf + From 13c91455426ea47f2b4378badbd58fd795a7e6f2 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 15 Sep 2023 00:47:57 +0200 Subject: [PATCH 54/70] add get submission all including all properties test --- .../test_ontology_submissions_controller.rb | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index ee8576bd..437b9180 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -210,4 +210,114 @@ def test_submissions_pagination submissions = MultiJson.load(last_response.body) assert_equal 1, submissions["collection"].length end + + + def test_submissions_default_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + + submission_default_attributes = LinkedData::Models::OntologySubmission.hypermedia_settings[:serialize_default].map(&:to_s) + + get("/submissions?display_links=false&display_context=false&include_status=ANY") + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + end + + def test_submissions_all_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + def submission_all_attributes + attrs = OntologySubmission.goo_attrs_to_load([:all]) + embed_attrs = attrs.select { |x| x.is_a?(Hash) }.first + + attrs.delete_if { |x| x.is_a?(Hash) }.map(&:to_s) + embed_attrs.keys.map(&:to_s) + end + get("/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_custom_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + include = 'ontology,contact,submissionId' + + get("/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_param_include + skip('only for local development regrouping a set of tests') + test_submissions_default_includes + test_submissions_all_includes + test_submissions_custom_includes + end + + private + def submission_keys(sub) + sub.to_hash.keys - %w[@id @type id] + end end From 62c7b0e54b1e6a517d2be1e4d2deb134625a1ade Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 15 Sep 2023 02:37:41 +0200 Subject: [PATCH 55/70] extract and use submission_include_params where we use submission.bring --- controllers/admin_controller.rb | 4 ++-- controllers/ontologies_controller.rb | 23 ++++--------------- .../ontology_submissions_controller.rb | 2 +- helpers/submission_helper.rb | 12 ++++++++++ 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/controllers/admin_controller.rb b/controllers/admin_controller.rb index 7ae6d800..747def93 100644 --- a/controllers/admin_controller.rb +++ b/controllers/admin_controller.rb @@ -68,7 +68,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil? check_last_modified(latest) - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + latest.bring(*submission_include_params) NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions) halt 204 end @@ -84,7 +84,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) end check_last_modified(latest) if latest - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + latest.bring(*submission_include_params) if latest reply(latest || {}) end diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index 1a8f2ebf..58518420 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -38,25 +38,12 @@ class OntologiesController < ApplicationController else latest = ont.latest_submission(status: :any) end - check_last_modified(latest) if latest - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) + if latest - if includes_param.first == :all - # Bring what we need to display all attr of the submission - latest.bring_remaining - latest.bring({:contact=>[:name, :email], - :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, - :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) - else - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << {:contact=>[:name, :email]} if includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} - - latest.bring(*includes) - end + check_last_modified(latest) + latest.bring(*submission_include_params) end - #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + reply(latest || {}) end @@ -66,7 +53,7 @@ class OntologiesController < ApplicationController patch '/:acronym/latest_submission' do ont = Ontology.find(params["acronym"]).first error 422, "You must provide an existing `acronym` to patch" if ont.nil? - + submission = ont.latest_submission(status: :any) submission.bring(*OntologySubmission.attributes) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 96411a8d..d88e2f43 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -60,7 +60,7 @@ class OntologySubmissionsController < ApplicationController ont.bring(:submissions) ont_submission = ont.submission(params["ontology_submission_id"]) error 404, "`submissionId` not found" if ont_submission.nil? - ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + ont_submission.bring(*submission_include_params) reply ont_submission end diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 07e4f27a..c23bcdaa 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -3,6 +3,18 @@ module Sinatra module Helpers module SubmissionHelper + def submission_include_params + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + includes = OntologySubmission.goo_attrs_to_load(includes_param) + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} + end + + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} + includes << {:contact=>[:name, :email]} + end + includes + end def retrieve_submissions(options) status = (options[:status] || "RDF").to_s.upcase From 5bf6a05862e2a36ad7fc1d16b96b0961fee900c5 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 15 Sep 2023 02:38:54 +0200 Subject: [PATCH 56/70] use retrieve_submissions helper in the :acronym/submissions endpoint --- .../ontology_submissions_controller.rb | 20 ++++++---------- helpers/application_helper.rb | 2 -- helpers/submission_helper.rb | 23 +++++-------------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index d88e2f43..5fd5218f 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -29,19 +29,13 @@ class OntologySubmissionsController < ApplicationController error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) check_access(ont) - if includes_param.first == :all - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) - ont.bring(submissions: [:released, :creationDate, :status, :submissionId, - {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus], :metrics =>[:classes, :individuals, :properties]) - - ont.submissions.each do |sub| - sub.bring_remaining - end - else - ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param)) - end - reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY") + } + subs = retrieve_submissions(options) + + reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId end ## diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 84c26497..cde04a34 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -359,8 +359,6 @@ def retrieve_latest_submissions(options = {}) latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| - # To retrieve all metadata, but slow when a lot of ontologies - sub.bring_remaining if includes_param.first == :all unless page? next if include_ready?(options) && !sub.ready? next if sub.ontology.nil? diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index c23bcdaa..b9a960b9 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -19,6 +19,7 @@ def submission_include_params def retrieve_submissions(options) status = (options[:status] || "RDF").to_s.upcase status = "RDF" if status.eql?("READY") + ontology_acronym = options[:ontology] any = status.eql?("ANY") include_views = options[:also_include_views] || false includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) @@ -32,28 +33,16 @@ def retrieve_submissions(options) submissions_query = submissions_query.where({submissionStatus: [ code: status]}) end + + submissions_query.where(ontology: [acronym: ontology_acronym]) if ontology_acronym + + submissions_query = apply_filters(submissions_query) submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], - :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, - :submissionStatus] - else - if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} - end - - if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} - includes << {:contact=>[:name, :email]} - end - end - submissions = submissions_query.include(includes) + submissions = submissions_query.include(submission_include_params) if page? submissions.page(page, size).all else From ea900272bec9ac6a8fc34211248bf1ba7510d167 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 21 Sep 2023 19:37:41 +0200 Subject: [PATCH 57/70] update Goo version and add submissions filters test --- Gemfile.lock | 12 +- .../test_ontology_submissions_controller.rb | 133 ++++++++++++++++++ 2 files changed, 140 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4a35ef0a..dde7e698 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 0c0dba92e28fd8c8261db1b3183946e0cf183b53 + revision: 0019b4b7b6e18be15fe068084510c03b674d23d8 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 1204ede68ed0a5af5e3fb355172496d5e0134544 + revision: 4c89c8346766d23e09b24c8e29750bf3a91e6b53 branch: development specs: ontologies_linked_data (0.0.1) @@ -110,6 +110,7 @@ GEM backports (3.24.1) bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) + benchmark-ips (2.12.0) bigdecimal (1.4.2) builder (3.2.4) capistrano (3.17.3) @@ -173,7 +174,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.8.0) + googleauth (1.8.1) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) @@ -225,7 +226,7 @@ GEM timeout net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) - net-smtp (0.3.3) + net-smtp (0.4.0) net-protocol net-ssh (7.2.0) netrc (0.11.0) @@ -354,6 +355,7 @@ PLATFORMS DEPENDENCIES activesupport (~> 3.1) bcrypt_pbkdf (>= 1.0, < 2.0) + benchmark-ips (~> 2.12) bigdecimal (= 1.4.2) capistrano (~> 3) capistrano-bundler @@ -405,4 +407,4 @@ DEPENDENCIES webmock BUNDLED WITH - 2.3.23 + 2.4.12 diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 7d4ed7ef..58359451 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -217,6 +217,139 @@ def test_submissions_pagination assert_equal 1, submissions["collection"].length end + def test_submissions_pagination_filter + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: 10, submission_count: 1) + group1 = LinkedData::Models::Group.new(acronym: 'group-1', name: "Test Group 1").save + group2 = LinkedData::Models::Group.new(acronym: 'group-2', name: "Test Group 2").save + category1 = LinkedData::Models::Category.new(acronym: 'category-1', name: "Test Category 1").save + category2 = LinkedData::Models::Category.new(acronym: 'category-2', name: "Test Category 2").save + + ontologies1 = ontologies[0..5].each do |o| + o.bring_remaining + o.group = [group1] + o.hasDomain = [category1] + o.save + end + + ontologies2 = ontologies[6..8].each do |o| + o.bring_remaining + o.group = [group2] + o.hasDomain = [category2] + o.save + end + + + + # test filter by group and category + get "/submissions?page=1&pagesize=100&group=#{group1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&group=#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category2.acronym}&group=#{group1.acronym}" + assert last_response.ok? + assert_equal 0, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category2.acronym}&group=#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + ontologies3 = ontologies[9] + ontologies3.bring_remaining + ontologies3.group = [group1, group2] + ontologies3.hasDomain = [category1, category2] + ontologies3.name = "name search test" + ontologies3.save + + # test search with acronym + [ + [ 1, ontologies.first.acronym], + [ 1, ontologies.last.acronym], + [ontologies.size, 'TEST-ONT'] + ].each do |count, acronym_search| + get "/submissions?page=1&pagesize=100&acronym=#{acronym_search}" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal count, submissions["collection"].length + end + + + # test search with name + [ + [ 1, ontologies.first.name], + [ 1, ontologies.last.name], + [ontologies.size - 1, 'TEST-ONT'] + ].each do |count, name_search| + get "/submissions?page=1&pagesize=100&name=#{name_search}" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + binding.pry unless submissions["collection"].length.eql?(count) + assert_equal count, submissions["collection"].length + end + + # test search with name and acronym + # search by name + get "/submissions?page=1&pagesize=100&name=search&acronym=search" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + # search by acronym + get "/submissions?page=1&pagesize=100&name=9&acronym=9" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + # search by acronym or name + get "/submissions?page=1&pagesize=100&name=search&acronym=8" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 2, submissions["collection"].length + + ontologies.first.name = "sort by test" + ontologies.first.save + sub = ontologies.first.latest_submission(status: :any).bring_remaining + sub.creationDate = DateTime.yesterday.to_datetime + sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first + sub.save + + #test search with sort + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&order_by=ontology_name" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.map{|x| x.name}.sort, submissions["collection"].map{|x| x["ontology"]["name"]} + + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&order_by=creationDate" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.map{|x| x.latest_submission(status: :any).bring(:creationDate).creationDate}.sort.map(&:to_s), submissions["collection"].map{|x| x["creationDate"]}.reverse + + # test search with format + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&hasOntologyLanguage=SKOS" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal 1, submissions["collection"].size + + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&hasOntologyLanguage=OWL" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.size-1 , submissions["collection"].size + + # test ontology filter with submission filter attributes + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&group=group-2&category=category-2&hasOntologyLanguage=OWL" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies2.size + 1 , submissions["collection"].size + end def test_submissions_default_includes ontology_count = 5 From d8c8e5bb3a6d441ab0afacd72d0ba3597e098382 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 21 Sep 2023 19:43:40 +0200 Subject: [PATCH 58/70] Fix: display contact for get submissions (#45) * add get submission all including all properties test * extract and use submission_include_params where we use submission.bring * use retrieve_submissions helper in the :acronym/submissions endpoint --- Gemfile.lock | 18 ++- controllers/admin_controller.rb | 4 +- controllers/ontologies_controller.rb | 20 +-- .../ontology_submissions_controller.rb | 21 ++-- helpers/application_helper.rb | 2 - helpers/submission_helper.rb | 35 +++--- .../test_ontology_submissions_controller.rb | 118 +++++++++++++++++- 7 files changed, 158 insertions(+), 60 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 44537959..4a35ef0a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 83e835de368bc9f19da800a477982e0ad770900d + revision: f440ae855a217807fead1d20629a0f187997b973 branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 1d78bde5a711d05475da0459308c7db074af5e21 + revision: 0c0dba92e28fd8c8261db1b3183946e0cf183b53 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 4c89c8346766d23e09b24c8e29750bf3a91e6b53 + revision: 1204ede68ed0a5af5e3fb355172496d5e0134544 branch: development specs: ontologies_linked_data (0.0.1) @@ -173,10 +173,9 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.7.0) + googleauth (1.8.0) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) - memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) @@ -205,7 +204,6 @@ GEM net-imap net-pop net-smtp - memoist (0.16.2) method_source (1.0.0) mime-types (3.5.1) mime-types-data (~> 3.2015) @@ -231,7 +229,7 @@ GEM net-protocol net-ssh (7.2.0) netrc (0.11.0) - newrelic_rpm (9.4.2) + newrelic_rpm (9.5.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -248,7 +246,7 @@ GEM rack (>= 0.4) rack-attack (6.6.1) rack (>= 1.0, < 3) - rack-cache (1.14.0) + rack-cache (1.13.0) rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) @@ -294,7 +292,7 @@ GEM rubyzip (2.3.2) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) - signet (0.17.0) + signet (0.18.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -326,7 +324,7 @@ GEM net-ssh (>= 2.8.0) systemu (2.6.5) temple (0.10.2) - tilt (2.2.0) + tilt (2.3.0) timeout (0.4.0) trailblazer-option (0.1.2) tzinfo (2.0.6) diff --git a/controllers/admin_controller.rb b/controllers/admin_controller.rb index 7ae6d800..747def93 100644 --- a/controllers/admin_controller.rb +++ b/controllers/admin_controller.rb @@ -68,7 +68,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil? check_last_modified(latest) - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + latest.bring(*submission_include_params) NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions) halt 204 end @@ -84,7 +84,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) end check_last_modified(latest) if latest - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + latest.bring(*submission_include_params) if latest reply(latest || {}) end diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index 99c0ce68..58518420 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -38,22 +38,12 @@ class OntologiesController < ApplicationController else latest = ont.latest_submission(status: :any) end - check_last_modified(latest) if latest - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) - if latest - if includes_param.first == :all - # Bring what we need to display all attr of the submission - latest.bring_remaining - latest.bring(*submission_attributes_all) - else - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << {:contact=>[:name, :email]} if includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} - latest.bring(*includes) - end + if latest + check_last_modified(latest) + latest.bring(*submission_include_params) end - #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + reply(latest || {}) end @@ -63,7 +53,7 @@ class OntologiesController < ApplicationController patch '/:acronym/latest_submission' do ont = Ontology.find(params["acronym"]).first error 422, "You must provide an existing `acronym` to patch" if ont.nil? - + submission = ont.latest_submission(status: :any) submission.bring(*OntologySubmission.attributes) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 7de7a3dc..77302c8d 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -29,17 +29,14 @@ class OntologySubmissionsController < ApplicationController error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) check_access(ont) - if includes_param.first == :all - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) - ont.bring(submission_attributes_all) - - ont.submissions.each do |sub| - sub.bring_remaining - end - else - ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param)) - end - reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY"), + ontology: params["acronym"] + } + subs = retrieve_submissions(options) + + reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId end ## @@ -58,7 +55,7 @@ class OntologySubmissionsController < ApplicationController ont.bring(:submissions) ont_submission = ont.submission(params["ontology_submission_id"]) error 404, "`submissionId` not found" if ont_submission.nil? - ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + ont_submission.bring(*submission_include_params) reply ont_submission end diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 5d6d1b0a..dbddbc5b 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -363,8 +363,6 @@ def retrieve_latest_submissions(options = {}) latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| - # To retrieve all metadata, but slow when a lot of ontologies - sub.bring_remaining if includes_param.first == :all unless page? next if include_ready?(options) && !sub.ready? next if sub.ontology.nil? diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index c1ee5dd3..07f82138 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -3,6 +3,18 @@ module Sinatra module Helpers module SubmissionHelper + def submission_include_params + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + includes = OntologySubmission.goo_attrs_to_load(includes_param) + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} + end + + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} + includes << {:contact=>[:name, :email]} + end + includes + end def submission_attributes_all out = [LinkedData::Models::OntologySubmission.embed_values_hash] @@ -16,14 +28,17 @@ def submission_attributes_all def retrieve_submissions(options) status = (options[:status] || "RDF").to_s.upcase status = "RDF" if status.eql?("READY") + ontology_acronym = options[:ontology] any = status.eql?("ANY") include_views = options[:also_include_views] || false includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) includes << :submissionStatus unless includes.include?(:submissionStatus) submissions_query = LinkedData::Models::OntologySubmission + submissions_query = submissions_query.where(ontology: [acronym: ontology_acronym]) if ontology_acronym + if any - submissions_query = submissions_query.where + submissions_query = submissions_query.where unless ontology_acronym else submissions_query = submissions_query.where({ submissionStatus: [code: status] }) end @@ -32,24 +47,8 @@ def retrieve_submissions(options) submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, { :contact => [:name, :email], - :ontology => [:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], - :submissionStatus => [:code], :hasOntologyLanguage => [:acronym], :metrics => [:classes, :individuals, :properties] }, - :submissionStatus] - else - if includes.find { |v| v.is_a?(Hash) && v.keys.include?(:ontology) } - includes << { :ontology => [:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain, :notes, :reviews, :projects, :acl, :viewOf] } - end - - if includes.find { |v| v.is_a?(Hash) && v.keys.include?(:contact) } - includes << { :contact => [:name, :email] } - end - end - submissions = submissions_query.include(includes) + submissions = submissions_query.include(submission_include_params) if page? submissions.page(page, size).all else diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 9ee81257..7d4ed7ef 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -39,6 +39,12 @@ def self._create_onts ont.save end + def setup + delete_ontologies_and_submissions + ont = Ontology.new(acronym: @@acronym, name: @@name, administeredBy: [@@user]) + ont.save + end + def test_submissions_for_given_ontology num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 1) ontology = created_ont_acronyms.first @@ -196,7 +202,7 @@ def test_download_acl_only end def test_submissions_pagination - num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 2, submission_count: 2) + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: 2, submission_count: 2) get "/submissions" assert last_response.ok? @@ -210,4 +216,114 @@ def test_submissions_pagination submissions = MultiJson.load(last_response.body) assert_equal 1, submissions["collection"].length end + + + def test_submissions_default_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + + submission_default_attributes = LinkedData::Models::OntologySubmission.hypermedia_settings[:serialize_default].map(&:to_s) + + get("/submissions?display_links=false&display_context=false&include_status=ANY") + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + end + + def test_submissions_all_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + def submission_all_attributes + attrs = OntologySubmission.goo_attrs_to_load([:all]) + embed_attrs = attrs.select { |x| x.is_a?(Hash) }.first + + attrs.delete_if { |x| x.is_a?(Hash) }.map(&:to_s) + embed_attrs.keys.map(&:to_s) + end + get("/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_custom_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + include = 'ontology,contact,submissionId' + + get("/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_param_include + skip('only for local development regrouping a set of tests') + test_submissions_default_includes + test_submissions_all_includes + test_submissions_custom_includes + end + + private + def submission_keys(sub) + sub.to_hash.keys - %w[@id @type id] + end end From bd6494c6377234176d62b5acf91c5e19250ee2f3 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 21 Sep 2023 20:46:38 +0200 Subject: [PATCH 59/70] Fix: Submissions filters with order_by for the same attribute (#46) * add get submission all including all properties test * extract and use submission_include_params where we use submission.bring * use retrieve_submissions helper in the :acronym/submissions endpoint * update Goo version and add submissions filters test --- Gemfile.lock | 10 +- config/solr/property_search/enumsconfig.xml | 12 + .../mapping-ISOLatin1Accent.txt | 246 ++++ config/solr/property_search/schema.xml | 1179 +++++++++++++++ config/solr/property_search/solrconfig.xml | 1299 +++++++++++++++++ config/solr/solr.xml | 60 + config/solr/term_search/enumsconfig.xml | 12 + .../term_search/mapping-ISOLatin1Accent.txt | 246 ++++ config/solr/term_search/schema.xml | 1222 ++++++++++++++++ config/solr/term_search/solrconfig.xml | 1299 +++++++++++++++++ helpers/request_params_helper.rb | 1 - .../test_ontology_submissions_controller.rb | 133 ++ 12 files changed, 5714 insertions(+), 5 deletions(-) create mode 100644 config/solr/property_search/enumsconfig.xml create mode 100644 config/solr/property_search/mapping-ISOLatin1Accent.txt create mode 100644 config/solr/property_search/schema.xml create mode 100644 config/solr/property_search/solrconfig.xml create mode 100644 config/solr/solr.xml create mode 100644 config/solr/term_search/enumsconfig.xml create mode 100644 config/solr/term_search/mapping-ISOLatin1Accent.txt create mode 100644 config/solr/term_search/schema.xml create mode 100644 config/solr/term_search/solrconfig.xml diff --git a/Gemfile.lock b/Gemfile.lock index 4a35ef0a..99821e68 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 0c0dba92e28fd8c8261db1b3183946e0cf183b53 + revision: 0019b4b7b6e18be15fe068084510c03b674d23d8 branch: development specs: goo (0.0.2) @@ -110,6 +110,7 @@ GEM backports (3.24.1) bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) + benchmark-ips (2.12.0) bigdecimal (1.4.2) builder (3.2.4) capistrano (3.17.3) @@ -173,7 +174,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.8.0) + googleauth (1.8.1) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) @@ -225,7 +226,7 @@ GEM timeout net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) - net-smtp (0.3.3) + net-smtp (0.4.0) net-protocol net-ssh (7.2.0) netrc (0.11.0) @@ -354,6 +355,7 @@ PLATFORMS DEPENDENCIES activesupport (~> 3.1) bcrypt_pbkdf (>= 1.0, < 2.0) + benchmark-ips (~> 2.12) bigdecimal (= 1.4.2) capistrano (~> 3) capistrano-bundler @@ -405,4 +407,4 @@ DEPENDENCIES webmock BUNDLED WITH - 2.3.23 + 2.4.12 diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/property_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/property_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml new file mode 100644 index 00000000..20824ea6 --- /dev/null +++ b/config/solr/property_search/schema.xml @@ -0,0 +1,1179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/property_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/config/solr/solr.xml b/config/solr/solr.xml new file mode 100644 index 00000000..d9d089e4 --- /dev/null +++ b/config/solr/solr.xml @@ -0,0 +1,60 @@ + + + + + + + + ${solr.max.booleanClauses:500000} + ${solr.sharedLib:} + ${solr.allowPaths:} + + + + ${host:} + ${solr.port.advertise:0} + ${hostContext:solr} + + ${genericCoreNodeNames:true} + + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + + + ${socketTimeout:600000} + ${connTimeout:60000} + ${solr.shardsWhitelist:} + + + + + diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/term_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/term_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml new file mode 100644 index 00000000..fa95e127 --- /dev/null +++ b/config/solr/term_search/schema.xml @@ -0,0 +1,1222 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/term_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 45091042..842ee0a7 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -42,7 +42,6 @@ def apply_submission_filters(query) hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS], ontology_hasDomain_acronym: params[:hasDomain]&.split(',') , #%w[Crop Vue_francais], ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], - ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], hasFormalityLevel: params[:hasFormalityLevel]&.split(','), #["http://w3id.org/nkos/nkostype#thesaurus"], ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 7d4ed7ef..58359451 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -217,6 +217,139 @@ def test_submissions_pagination assert_equal 1, submissions["collection"].length end + def test_submissions_pagination_filter + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: 10, submission_count: 1) + group1 = LinkedData::Models::Group.new(acronym: 'group-1', name: "Test Group 1").save + group2 = LinkedData::Models::Group.new(acronym: 'group-2', name: "Test Group 2").save + category1 = LinkedData::Models::Category.new(acronym: 'category-1', name: "Test Category 1").save + category2 = LinkedData::Models::Category.new(acronym: 'category-2', name: "Test Category 2").save + + ontologies1 = ontologies[0..5].each do |o| + o.bring_remaining + o.group = [group1] + o.hasDomain = [category1] + o.save + end + + ontologies2 = ontologies[6..8].each do |o| + o.bring_remaining + o.group = [group2] + o.hasDomain = [category2] + o.save + end + + + + # test filter by group and category + get "/submissions?page=1&pagesize=100&group=#{group1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&group=#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category2.acronym}&group=#{group1.acronym}" + assert last_response.ok? + assert_equal 0, MultiJson.load(last_response.body)["collection"].length + get "/submissions?page=1&pagesize=100&hasDomain=#{category2.acronym}&group=#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + ontologies3 = ontologies[9] + ontologies3.bring_remaining + ontologies3.group = [group1, group2] + ontologies3.hasDomain = [category1, category2] + ontologies3.name = "name search test" + ontologies3.save + + # test search with acronym + [ + [ 1, ontologies.first.acronym], + [ 1, ontologies.last.acronym], + [ontologies.size, 'TEST-ONT'] + ].each do |count, acronym_search| + get "/submissions?page=1&pagesize=100&acronym=#{acronym_search}" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal count, submissions["collection"].length + end + + + # test search with name + [ + [ 1, ontologies.first.name], + [ 1, ontologies.last.name], + [ontologies.size - 1, 'TEST-ONT'] + ].each do |count, name_search| + get "/submissions?page=1&pagesize=100&name=#{name_search}" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + binding.pry unless submissions["collection"].length.eql?(count) + assert_equal count, submissions["collection"].length + end + + # test search with name and acronym + # search by name + get "/submissions?page=1&pagesize=100&name=search&acronym=search" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + # search by acronym + get "/submissions?page=1&pagesize=100&name=9&acronym=9" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + # search by acronym or name + get "/submissions?page=1&pagesize=100&name=search&acronym=8" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 2, submissions["collection"].length + + ontologies.first.name = "sort by test" + ontologies.first.save + sub = ontologies.first.latest_submission(status: :any).bring_remaining + sub.creationDate = DateTime.yesterday.to_datetime + sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first + sub.save + + #test search with sort + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&order_by=ontology_name" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.map{|x| x.name}.sort, submissions["collection"].map{|x| x["ontology"]["name"]} + + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&order_by=creationDate" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.map{|x| x.latest_submission(status: :any).bring(:creationDate).creationDate}.sort.map(&:to_s), submissions["collection"].map{|x| x["creationDate"]}.reverse + + # test search with format + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&hasOntologyLanguage=SKOS" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal 1, submissions["collection"].size + + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&hasOntologyLanguage=OWL" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.size-1 , submissions["collection"].size + + # test ontology filter with submission filter attributes + get "/submissions?page=1&pagesize=100&acronym=tes&name=tes&group=group-2&category=category-2&hasOntologyLanguage=OWL" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies2.size + 1 , submissions["collection"].size + end def test_submissions_default_includes ontology_count = 5 From 47ffdabdba040a7c6601f7b31658827eedceb77f Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 9 Oct 2023 10:43:27 +0200 Subject: [PATCH 60/70] make the ontology submissions endpoint include views --- controllers/ontology_submissions_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 77302c8d..0068a5f1 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -30,7 +30,7 @@ class OntologySubmissionsController < ApplicationController check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) check_access(ont) options = { - also_include_views: params["also_include_views"], + also_include_views: true, status: (params["include_status"] || "ANY"), ontology: params["acronym"] } From 9fb09a27b803bb758019b5e314d75d0d620a60e4 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Sun, 22 Oct 2023 11:38:59 +0200 Subject: [PATCH 61/70] include all metrics attribues in the submissions endpoints (#53) --- helpers/submission_helper.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 07f82138..b79737d0 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -13,6 +13,14 @@ def submission_include_params if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} includes << {:contact=>[:name, :email]} end + + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:metrics)} + includes << { metrics: [:maxChildCount, :properties, :classesWithMoreThan25Children, + :classesWithOneChild, :individuals, :maxDepth, :classes, + :classesWithNoDefinition, :averageChildCount, :numberOfAxioms, + :entities]} + end + includes end From 1602a3114538f3795fb522c106f2f117b935f160 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 1 Nov 2023 01:03:41 +0100 Subject: [PATCH 62/70] add ontology submissions filter by status (#56) --- helpers/request_params_helper.rb | 2 +- .../test_ontology_submissions_controller.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 842ee0a7..f1b8268c 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -45,9 +45,9 @@ def apply_submission_filters(query) isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], hasFormalityLevel: params[:hasFormalityLevel]&.split(','), #["http://w3id.org/nkos/nkostype#thesaurus"], ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + status: params[:status]&.split(','), #"retired", } inverse_filters = { - status: params[:status], #"retired", submissionStatus: params[:submissionStatus] #"RDF", } diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 58359451..147b9c12 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -313,6 +313,7 @@ def test_submissions_pagination_filter ontologies.first.name = "sort by test" ontologies.first.save sub = ontologies.first.latest_submission(status: :any).bring_remaining + sub.status = 'retired' sub.creationDate = DateTime.yesterday.to_datetime sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first sub.save @@ -349,6 +350,19 @@ def test_submissions_pagination_filter submissions = MultiJson.load(last_response.body) refute_empty submissions["collection"] assert_equal ontologies2.size + 1 , submissions["collection"].size + + # test ontology filter with status + get "/submissions?page=1&pagesize=100&status=retired" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal 1 , submissions["collection"].size + + get "/submissions?page=1&pagesize=100&status=alpha,beta,production" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.size - 1 , submissions["collection"].size end def test_submissions_default_includes From 01cdda43e6e9ed8f4590d98fa6a00842b05a9b32 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 3 Nov 2023 14:01:04 +0100 Subject: [PATCH 63/70] add agent usage attribute tests (#55) --- Gemfile | 2 +- Gemfile.lock | 21 ++++++++++++--------- test/controllers/test_agents_controller.rb | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Gemfile b/Gemfile index b4c0c5f7..ff82b939 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'activesupport', '~> 3.1' +gem 'activesupport', '~> 3.2' # see https://github.com/ncbo/ontologies_api/issues/69 gem 'bigdecimal', '1.4.2' gem 'faraday', '~> 1.9' diff --git a/Gemfile.lock b/Gemfile.lock index 8b9712c2..b438802c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: f440ae855a217807fead1d20629a0f187997b973 + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 69466682c1e9cb2c338539195013dbec9714ca7d + revision: 5979402d5138850fb9bdb34edfa350e9af1b5d22 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 087c28b92527a1df1665d25c38277456e9acf06e + revision: b49ad10781e83360a5dd1968972c4048592eefda branch: development specs: ontologies_linked_data (0.0.1) @@ -108,11 +108,12 @@ GEM airbrussh (1.5.0) sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) + base64 (0.1.1) bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.3) + capistrano (3.18.0) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -164,7 +165,7 @@ GEM ffi (~> 1.0) google-apis-analytics_v3 (0.13.0) google-apis-core (>= 0.11.0, < 2.a) - google-apis-core (0.11.1) + google-apis-core (0.11.2) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -216,7 +217,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.4.1) + net-imap (0.4.3) date net-protocol net-pop (0.1.2) @@ -229,7 +230,8 @@ GEM net-protocol net-ssh (7.2.0) netrc (0.11.0) - newrelic_rpm (9.5.0) + newrelic_rpm (9.6.0) + base64 oj (2.18.5) omni_logger (0.1.4) logger @@ -349,10 +351,11 @@ GEM PLATFORMS x86_64-darwin-21 + x86_64-darwin-23 x86_64-linux DEPENDENCIES - activesupport (~> 3.1) + activesupport (~> 3.2) bcrypt_pbkdf (>= 1.0, < 2.0) bigdecimal (= 1.4.2) capistrano (~> 3) @@ -405,4 +408,4 @@ DEPENDENCIES webmock BUNDLED WITH - 2.4.12 + 2.3.26 diff --git a/test/controllers/test_agents_controller.rb b/test/controllers/test_agents_controller.rb index ef0e5c47..e7521b37 100644 --- a/test/controllers/test_agents_controller.rb +++ b/test/controllers/test_agents_controller.rb @@ -28,14 +28,14 @@ def teardown end def test_all_agents - get '/agents' + get '/agents?display=all' assert last_response.ok? created_agents = MultiJson.load(last_response.body) - @agents.each do |agent| created_agent = created_agents.select{|x| x["name"].eql?(agent[:name])}.first refute_nil created_agent + refute_nil created_agent["usages"] assert_equal agent[:name], created_agent["name"] assert_equal agent[:identifiers].size, created_agent["identifiers"].size assert_equal agent[:identifiers].map{|x| x[:notation]}.sort, created_agent["identifiers"].map{|x| x['notation']}.sort From d8eb63a76125fbf2288373d00da340c29d4ff4ad Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 8 Nov 2023 13:00:14 +0100 Subject: [PATCH 64/70] Fix: optimize fetching all agents usages query by batch loading them (#57) * add agent usage attribute tests * optimize fetching all agents usages query by batch loading them --- Gemfile.lock | 16 ++++++++-------- controllers/agents_controller.rb | 5 +++++ test/controllers/test_agents_controller.rb | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b438802c..1212ff1b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: b49ad10781e83360a5dd1968972c4048592eefda + revision: 7d16b7b1ecc4da8bfd93db2a106737b06d84d6ff branch: development specs: ontologies_linked_data (0.0.1) @@ -108,7 +108,7 @@ GEM airbrussh (1.5.0) sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) - base64 (0.1.1) + base64 (0.2.0) bcrypt (3.1.19) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) @@ -131,7 +131,7 @@ GEM rexml cube-ruby (0.0.3) dante (0.2.0) - date (3.3.3) + date (3.3.4) declarative (0.0.20) docile (1.4.0) domain_name (0.5.20190701) @@ -196,8 +196,8 @@ GEM json_pure (2.6.3) jwt (2.7.1) kgio (2.11.4) - libxml-ruby (4.1.1) - logger (1.5.3) + libxml-ruby (4.1.2) + logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -217,12 +217,12 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.4.3) + net-imap (0.4.4) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) @@ -327,7 +327,7 @@ GEM systemu (2.6.5) temple (0.10.3) tilt (2.3.0) - timeout (0.4.0) + timeout (0.4.1) trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index 87572e99..1bf86321 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -14,6 +14,11 @@ class AgentsController < ApplicationController else agents = query.to_a end + + if includes_param.include?(:all) || includes_param.include?(:usages) + LinkedData::Models::Agent.load_agents_usages(agents) + end + reply agents end diff --git a/test/controllers/test_agents_controller.rb b/test/controllers/test_agents_controller.rb index e7521b37..de36bc36 100644 --- a/test/controllers/test_agents_controller.rb +++ b/test/controllers/test_agents_controller.rb @@ -28,12 +28,12 @@ def teardown end def test_all_agents - get '/agents?display=all' + get '/agents?display=all&page=1' assert last_response.ok? created_agents = MultiJson.load(last_response.body) @agents.each do |agent| - created_agent = created_agents.select{|x| x["name"].eql?(agent[:name])}.first + created_agent = created_agents["collection"].select{|x| x["name"].eql?(agent[:name])}.first refute_nil created_agent refute_nil created_agent["usages"] assert_equal agent[:name], created_agent["name"] From 1d13691e71bd29f56ccab4c0ded8f9f8897f569f Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 13 Nov 2023 09:36:25 +0100 Subject: [PATCH 65/70] Feature: Add ontologies_api docker image build CI (#58) * add docker build CI --- .github/workflows/docker-dev-release.yml | 45 +++++ Dockerfile | 3 +- Gemfile | 1 + Gemfile.lock | 3 +- config/environments/config.rb.sample | 200 +++++++++++------------ 5 files changed, 142 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/docker-dev-release.yml diff --git a/.github/workflows/docker-dev-release.yml b/.github/workflows/docker-dev-release.yml new file mode 100644 index 00000000..dda0554d --- /dev/null +++ b/.github/workflows/docker-dev-release.yml @@ -0,0 +1,45 @@ +name: Docker branch Images build + +on: + push: + branches: + - development + - stage + - test + +jobs: + push_to_registry: + name: Push Docker branch image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: agroportal/ontologies_api + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + build-args: | + RUBY_VERSION=2.7 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index 3e65fe4a..6294e102 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ ENV BUNDLE_PATH=/srv/ontoportal/bundle RUN bundle install COPY . /srv/ontoportal/ontologies_api +RUN cp /srv/ontoportal/ontologies_api/config/environments/config.rb.sample /srv/ontoportal/ontologies_api/config/environments/development.rb EXPOSE 9393 -CMD ["bundle", "exec", "rackup", "-p", "9393", "--host", "0.0.0.0"] +CMD ["bundle", "exec", "rackup", "-p", "9393", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Gemfile b/Gemfile index ff82b939..5716331d 100644 --- a/Gemfile +++ b/Gemfile @@ -63,6 +63,7 @@ group :development do gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo' end + group :profiling do gem 'rack-mini-profiler' end diff --git a/Gemfile.lock b/Gemfile.lock index 1212ff1b..8ff9e209 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -350,7 +350,6 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-darwin-23 x86_64-linux @@ -408,4 +407,4 @@ DEPENDENCIES webmock BUNDLED WITH - 2.3.26 + 2.4.21 diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index e5f9fd9c..45ebf2be 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -3,120 +3,106 @@ # development.rb # test.rb -begin - LinkedData.config do |config| - config.repository_folder = "/srv/ncbo/repository" - config.goo_host = "localhost" - config.goo_port = 9000 - config.search_server_url = "http://localhost:8082/solr/term_search_core1" - config.property_search_server_url = "http://localhost:8082/solr/prop_search_core1" - config.rest_url_prefix = "http://#{$SITE_URL}:8080/" - config.replace_url_prefix = true - config.enable_security = true - - config.apikey = "24e0e77e-54e0-11e0-9d7b-005056aa3316" - config.ui_host = "http://#{$SITE_URL}" - config.enable_monitoring = false - config.cube_host = "localhost" - config.enable_resource_index = false - - # Used to define other BioPortal to which this appliance can be mapped to - # Example to map to the NCBO BioPortal : {"ncbo" => {"api" => "http://data.bioontology.org", "ui" => "http://bioportal.bioontology.org", "apikey" => ""}} - # Then create the mapping using the following class in JSON : "http://purl.bioontology.org/ontology/MESH/C585345": "ncbo:MESH" - # Where "ncbo" is the key in the interportal_hash. Use only lowercase letters for this key. - # And do not use "ext" as a key, it is reserved for clases outside of any BioPortal - config.interportal_hash = {} - - # Caches - config.http_redis_host = "localhost" - config.http_redis_port = 6380 - config.enable_http_cache = true - config.goo_redis_host = "localhost" - config.goo_redis_port = 6382 +GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store" +GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" +GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/data/" +GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" +GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" +GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 +MGREP_HOST = ENV.include?("MGREP_HOST") ? ENV["MGREP_HOST"] : "localhost" +MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55555 +MGREP_DICTIONARY_FILE = ENV.include?("MGREP_DICTIONARY_FILE") ? ENV["MGREP_DICTIONARY_FILE"] : "./test/data/dictionary.txt" +REDIS_GOO_CACHE_HOST = ENV.include?("REDIS_GOO_CACHE_HOST") ? ENV["REDIS_GOO_CACHE_HOST"] : "localhost" +REDIS_HTTP_CACHE_HOST = ENV.include?("REDIS_HTTP_CACHE_HOST") ? ENV["REDIS_HTTP_CACHE_HOST"] : "localhost" +REDIS_PERSISTENT_HOST = ENV.include?("REDIS_PERSISTENT_HOST") ? ENV["REDIS_PERSISTENT_HOST"] : "localhost" +REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 +REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/ontologies_report.json" +REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" +REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : "http://localhost:9393" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" - Goo.use_cache = true - - # Email notifications - config.enable_notifications = false - config.email_sender = "admin@example.org" # Default sender for emails - config.email_override = "override@example.org" # all email gets sent here. Disable with email_override_disable. - config.email_disable_override = true - config.smtp_host = "localhost" - config.smtp_port = 25 - config.smtp_auth_type = :none # :none, :plain, :login, :cram_md5 - config.smtp_domain = "example.org" - # Emails of the instance administrators to get mail notifications when new user or new ontology - config.admin_emails = ["admin@example.org"] +begin + # For prefLabel extract main_lang first, or anything if no main found. + # For other properties only properties with a lang that is included in main_lang are used + Goo.main_languages = ["en", "fr"] + Goo.use_cache = false +rescue NoMethodError + puts "(CNFG) >> Goo.main_lang not available" +end - # PURL server config parameters - config.enable_purl = false - config.purl_host = "purl.example.org" - config.purl_port = 80 - config.purl_username = "admin" - config.purl_password = "password" - config.purl_maintainers = "admin" - config.purl_target_url_prefix = "http://example.org" +LinkedData.config do |config| + config.goo_backend_name = GOO_BACKEND_NAME.to_s + config.goo_host = GOO_HOST.to_s + config.goo_port = GOO_PORT.to_i + config.goo_path_query = GOO_PATH_QUERY.to_s + config.goo_path_data = GOO_PATH_DATA.to_s + config.goo_path_update = GOO_PATH_UPDATE.to_s + config.goo_redis_host = REDIS_GOO_CACHE_HOST.to_s + config.goo_redis_port = REDIS_PORT.to_i + config.http_redis_host = REDIS_HTTP_CACHE_HOST.to_s + config.http_redis_port = REDIS_PORT.to_i + config.ontology_analytics_redis_host = REDIS_PERSISTENT_HOST.to_s + config.ontology_analytics_redis_port = REDIS_PORT.to_i + config.search_server_url = SOLR_TERM_SEARCH_URL.to_s + config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s + config.replace_url_prefix = true + config.rest_url_prefix = REST_URL_PREFIX.to_s +# config.enable_notifications = false - # Ontology Google Analytics Redis - # disabled - config.ontology_analytics_redis_host = "localhost" - config.enable_ontology_analytics = false - config.ontology_analytics_redis_port = 6379 - end -rescue NameError - puts "(CNFG) >> LinkedData not available, cannot load config" + config.interportal_hash = { + "agroportal" => { + "api" => "http://data.agroportal.lirmm.fr", + "ui" => "http://agroportal.lirmm.fr", + "apikey" => "1cfae05f-9e67-486f-820b-b393dec5764b" + }, + "ncbo" => { + "api" => "http://data.bioontology.org", + "apikey" => "4a5011ea-75fa-4be6-8e89-f45c8c84844e", + "ui" => "http://bioportal.bioontology.org", + }, + "sifr" => { + "api" => "http://data.bioportal.lirmm.fr", + "ui" => "http://bioportal.lirmm.fr", + "apikey" => "1cfae05f-9e67-486f-820b-b393dec5764b" + } + } + config.oauth_providers = { + github: { + check: :access_token, + link: 'https://api.github.com/user' + }, + keycloak: { + check: :jwt_token, + cert: 'KEYCLOAK_SECRET_KEY' + }, + orcid: { + check: :access_token, + link: 'https://pub.orcid.org/v3.0/me' + }, + google: { + check: :access_token, + link: 'https://www.googleapis.com/oauth2/v3/userinfo' + } + } end -begin - Annotator.config do |config| - config.mgrep_dictionary_file = "/srv/mgrep/dictionary/dictionary.txt" - config.stop_words_default_file = "./config/default_stop_words.txt" - config.mgrep_host = "localhost" - config.mgrep_port = 55555 - config.mgrep_alt_host = "localhost" - config.mgrep_alt_port = 55555 - config.annotator_redis_host = "localhost" - config.annotator_redis_port = 6379 - end -rescue NameError - puts "(CNFG) >> Annotator not available, cannot load config" +Annotator.config do |config| + config.annotator_redis_host = REDIS_PERSISTENT_HOST.to_s + config.annotator_redis_port = REDIS_PORT.to_i + config.mgrep_host = MGREP_HOST.to_s + config.mgrep_port = MGREP_PORT.to_i + config.mgrep_dictionary_file = MGREP_DICTIONARY_FILE.to_s end LinkedData::OntologiesAPI.config do |config| - config.restrict_download = ["ACR0", "ACR1", "ACR2"] -end - -begin - LinkedData::OntologiesAPI.config do |config| - config.enable_unicorn_workerkiller = true - config.enable_throttling = false - config.enable_monitoring = false - config.cube_host = "localhost" - config.http_redis_host = "localhost" - config.http_redis_port = 6380 - config.ontology_rank = "" - config.resolver_redis_host = "localhost" - config.resolver_redis_port = 6379 - config.restrict_download = ["ACR0", "ACR1", "ACR2"] - end -rescue NameError - puts "(CNFG) >> OntologiesAPI not available, cannot load config" + config.http_redis_host = REDIS_HTTP_CACHE_HOST.to_s + config.http_redis_port = REDIS_PORT.to_i +# config.restrict_download = ["ACR0", "ACR1", "ACR2"] end -begin - NcboCron.config do |config| - config.redis_host = Annotator.settings.annotator_redis_host - config.redis_port = Annotator.settings.annotator_redis_port - config.enable_ontology_analytics = false - config.enable_ontologies_report = false - # Schedulues - config.cron_schedule = "30 */4 * * *" - # Pull schedule - config.pull_schedule = "00 18 * * *" - # Pull long schedule for ontology that are pulled less frequently: run weekly on monday at 11 a.m. (23:00) - config.pull_schedule_long = "00 23 * * 1" - config.pull_long_ontologies = ["BIOREFINERY", "TRANSMAT", "GO"] - end -rescue NameError - puts "(CNFG) >> NcboCron not available, cannot load config" -end +NcboCron.config do |config| + config.redis_host = REDIS_PERSISTENT_HOST.to_s + config.redis_port = REDIS_PORT.to_i + config.ontology_report_path = REPORT_PATH +end \ No newline at end of file From 0e559c539fbd5f7dbf10729743aad377f3f9a63f Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 22 Nov 2023 02:38:51 +0100 Subject: [PATCH 66/70] Feature: add ontoportal bash script (#59) * add ontoportal bash script to run test and development servers * update README.md * update docker CI to work in production releases --- .env.sample | 1 + ...ocker-dev-release.yml => docker-image.yml} | 20 ++- .github/workflows/ruby-unit-tests.yml | 2 + .gitignore | 2 + Gemfile.lock | 16 +- README.md | 62 ++++++- bin/ontoportal | 166 ++++++++++++++++++ config/environments/config.rb.sample | 2 +- docker-compose.yml | 9 +- 9 files changed, 250 insertions(+), 30 deletions(-) create mode 100644 .env.sample rename .github/workflows/{docker-dev-release.yml => docker-image.yml} (64%) create mode 100755 bin/ontoportal diff --git a/.env.sample b/.env.sample new file mode 100644 index 00000000..719949b1 --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +API_URL=http://localhost:9393 \ No newline at end of file diff --git a/.github/workflows/docker-dev-release.yml b/.github/workflows/docker-image.yml similarity index 64% rename from .github/workflows/docker-dev-release.yml rename to .github/workflows/docker-image.yml index dda0554d..0368c3ac 100644 --- a/.github/workflows/docker-dev-release.yml +++ b/.github/workflows/docker-image.yml @@ -6,7 +6,8 @@ on: - development - stage - test - + release: + types: [ published ] jobs: push_to_registry: name: Push Docker branch image to Docker Hub @@ -22,24 +23,33 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: - images: agroportal/ontologies_api + images: | + agroportal/ontologies_api + ghcr.io/${{ github.repository }} - name: Build and push Docker image uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64, linux/arm64 build-args: | - RUBY_VERSION=2.7 + RUBY_VERSION=2.7.8 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 6b2c973d..e7b3524f 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: copy-env-config + run: cp .env.sample .env - name: Build docker-compose run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step - name: Run unit tests diff --git a/.gitignore b/.gitignore index 886a220f..8b568832 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,5 @@ test/data/ontology_files/catalog-v001.xml create_permissions.log ontologies_api.iml + +.env diff --git a/Gemfile.lock b/Gemfile.lock index 8ff9e209..9875b7fc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 7d16b7b1ecc4da8bfd93db2a106737b06d84d6ff + revision: 4c7dfa80a8bb4a7d8cfb7ad1fc8a1a88e420e59e branch: development specs: ontologies_linked_data (0.0.1) @@ -109,7 +109,7 @@ GEM sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) base64 (0.2.0) - bcrypt (3.1.19) + bcrypt (3.1.20) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) @@ -134,8 +134,7 @@ GEM date (3.3.4) declarative (0.0.20) docile (1.4.0) - domain_name (0.5.20190701) - unf (>= 0.0.5, < 1.0.0) + domain_name (0.6.20231109) ed25519 (1.3.0) faraday (1.10.3) faraday-em_http (~> 1.0) @@ -217,7 +216,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.4.4) + net-imap (0.4.6) date net-protocol net-pop (0.1.2) @@ -242,7 +241,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.3) + public_suffix (5.0.4) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -321,7 +320,7 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.5) + sshkit (1.21.6) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) @@ -332,9 +331,6 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.2) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) diff --git a/README.md b/README.md index dfaa77ea..b4caa10a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,53 @@ ontologies_api provides a RESTful interface for accessing [BioPortal](https://bioportal.bioontology.org/) (an open repository of biomedical ontologies). Supported services include downloads, search, access to terms and concepts, text annotation, and much more. -## Prerequisites +# Run ontologies_api + +## Using OntoPortal api utilities script +### See help + +```bash +bin/ontoportal help +``` + +``` +Usage: bin/ontoportal {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY] + dev : Start the Ontoportal API development server. + Example: bin/ontoportal dev --api-url http://localhost:9393 + Use --reset-cache to remove volumes: bin/ontoportal dev --reset-cache + test : Run tests. + run : Run a command in the Ontoportal API Docker container. + help : Show this help message. + +Description: + This script provides convenient commands for managing an Ontoportal API + application using Docker Compose. It includes options for starting the development server, + running tests, and executing commands within the Ontoportal API Docker container. + +Goals: + - Simplify common tasks related to Ontoportal API development using Docker. + - Provide a consistent and easy-to-use interface for common actions. + + +``` +### Configuration +``` +cp .env.sample .env +``` + +### Run dev +```bash +bin/ontoportal dev +``` + +### Run test with a local OntoPortal API +```bash +bin/ontoportal test +``` + + +## Manually +### Prerequisites - [Ruby 2.x](http://www.ruby-lang.org/en/downloads/) (most recent patch level) - [rbenv](https://github.com/sstephenson/rbenv) and [ruby-build](https://github.com/sstephenson/ruby-build) (optional) @@ -19,7 +65,7 @@ ontologies_api provides a RESTful interface for accessing [BioPortal](https://bi - [Solr](http://lucene.apache.org/solr/) - BioPortal indexes ontology class and property content using Solr (a Lucene-based server) -## Configuring Solr +### Configuring Solr To configure Solr for ontologies_api usage, modify the example project included with Solr by doing the following: @@ -46,22 +92,22 @@ To configure Solr for ontologies_api usage, modify the example project included # Edit the ontologieS_api/config/environments/{env}.rb file to point to your running instance: # http://localhost:8983/solr/NCBO1 -## Installing +### Installing -### Clone the repository +#### Clone the repository ``` $ git clone git@github.com:ncbo/ontologies_api.git $ cd ontologies_api ``` -### Install the dependencies +#### Install the dependencies ``` $ bundle install ``` -### Create an environment configuration file +#### Create an environment configuration file ``` $ cp config/environments/config.rb.sample config/environments/development.rb @@ -73,7 +119,7 @@ production.rb
development.rb
test.rb -### Run the unit tests (optional) +#### Run the unit tests (optional) Requires a configuration file for the test environment: @@ -87,7 +133,7 @@ Execute the suite of tests from the command line: $ bundle exec rake test ``` -### Run the application +#### Run the application ``` $ bundle exec rackup --port 9393 diff --git a/bin/ontoportal b/bin/ontoportal new file mode 100755 index 00000000..573b49c7 --- /dev/null +++ b/bin/ontoportal @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Function to display script usage information +show_help() { + echo "Usage: $0 {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY]" + echo " dev : Start the Ontoportal API development server." + echo " Example: $0 dev --api-url http://localhost:9393" + echo " Use --reset-cache to remove volumes: $0 dev --reset-cache" + echo " test : Run tests." + echo " run : Run a command in the Ontoportal API Docker container." + echo " help : Show this help message." + echo + echo "Description:" + echo " This script provides convenient commands for managing an Ontoportal API" + echo " application using Docker Compose. It includes options for starting the development server," + echo " running tests, and executing commands within the Ontoportal API Docker container." + echo + echo "Goals:" + echo " - Simplify common tasks related to Ontoportal API development using Docker." + echo " - Provide a consistent and easy-to-use interface for common actions." +} +# Function to update or create the .env file with API_URL and API_KEY +update_env_file() { + local api_url="$1" + + # Update the .env file with the provided values + file_content=$(<.env) + + # Make changes to the variable + while IFS= read -r line; do + if [[ "$line" == "API_URL="* ]]; then + echo "API_URL=$api_url" + else + echo "$line" + fi + done <<< "$file_content" > .env +} + +# Function to create configuration files if they don't exist +create_config_files() { + if [ ! -f ".env" ]; then + echo "Creating .env file from env.sample" + cp .env.sample .env + fi + + if [ ! -f "config/environments/development.rb" ]; then + echo "Creating config/environments/development.rb file from config/environments/config.rb.sample" + cp config/bioportal_config_env.rb.sample config/bioportal_config_development.rb + fi +} + +# Function to handle the "dev" option +dev() { + echo "Starting Ontoportal API development server..." + + create_config_files + local reset_cache=false + local api_url="" + + + # Check for command line arguments + while [[ "$#" -gt 0 ]]; do + case $1 in + --reset-cache) + reset_cache=true + shift + ;; + --api-url) + api_url="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + + + # Check if arguments are provided + if [ -n "$api_url" ] ; then + # If arguments are provided, update the .env file + update_env_file "$api_url" + else + # If no arguments, fetch values from the .env file + source .env + api_url="$API_URL" + fi + + if [ -z "$api_url" ] ; then + echo "Error: Missing required arguments. Please provide both --api-url or update them in your .env" + exit 1 + fi + + # Check if --reset-cache is present and execute docker compose down --volumes + if [ "$reset_cache" = true ]; then + echo "Resetting cache. Running: docker compose down --volumes" + docker compose down --volumes + fi + + echo "Run: bundle exec api s -b 0.0.0.0 -p 3000" + docker compose run --rm -it --service-ports api bash -c "(bundle check || bundle install) && bundle exec rackup -o 0.0.0.0 --port 9393" +} + +# Function to handle the "test" option +test() { + + + local api_url="" + local test_path="" + local test_options="" + + # Check for command line arguments + while [ "$#" -gt 0 ]; do + case "$1" in + --api-url) + shift + api_url="$1" + ;; + *) + if [ -z "$test_path" ]; then + test_path="$1" + else + test_options="$test_options $1" + fi + ;; + esac + shift + done + + + + script="API_URL=$api_url bundle exec rake test TEST=\"$test_path\" TESTOPTS=\"$test_options\"" + echo "Running tests..." + echo "Run: $script" + + docker compose run --rm -it api bash -c "(bundle check || bundle install) && $script" +} + +# Function to handle the "run" option +run() { + echo "Run: $*" + docker compose run --rm -it api bash -c "$*" +} + +# Main script logic +case "$1" in + "run") + run "${@:2}" + ;; + "dev") + dev "${@:2}" + ;; + "test") + test "${@:2}" + ;; + "help") + show_help + ;; + *) + show_help + exit 1 + ;; +esac diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 45ebf2be..8713b9f2 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -18,7 +18,7 @@ REDIS_PERSISTENT_HOST = ENV.include?("REDIS_PERSISTENT_HOST") ? ENV["REDIS_PERSI REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/ontologies_report.json" REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" -REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : "http://localhost:9393" +REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : ENV["API_URL"] || "http://localhost:9393" SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" diff --git a/docker-compose.yml b/docker-compose.yml index 5cb64963..f7325381 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,5 @@ x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_api:0.0.1 + image: agroportal/ontologies_api:development environment: &env BUNDLE_PATH: /srv/ontoportal/bundle # default bundle config resolves to /usr/local/bundle/config inside of the container @@ -39,6 +34,8 @@ x-app: &app services: api: <<: *app + env_file: + .env environment: <<: *env GOO_BACKEND_NAME: 4store From 00967efabb097e51a67e753aec89286dba26d75e Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 22 Nov 2023 02:38:51 +0100 Subject: [PATCH 67/70] Feature: add ontoportal bash script (#59) * add ontoportal bash script to run test and development servers * update README.md * update docker CI to work in production releases --- .env.sample | 1 + ...ocker-dev-release.yml => docker-image.yml} | 20 ++- .github/workflows/ruby-unit-tests.yml | 4 +- .gitignore | 2 + Gemfile.lock | 16 +- README.md | 62 ++++++- bin/ontoportal | 166 ++++++++++++++++++ config/environments/config.rb.sample | 2 +- docker-compose.yml | 9 +- 9 files changed, 251 insertions(+), 31 deletions(-) create mode 100644 .env.sample rename .github/workflows/{docker-dev-release.yml => docker-image.yml} (64%) create mode 100755 bin/ontoportal diff --git a/.env.sample b/.env.sample new file mode 100644 index 00000000..719949b1 --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +API_URL=http://localhost:9393 \ No newline at end of file diff --git a/.github/workflows/docker-dev-release.yml b/.github/workflows/docker-image.yml similarity index 64% rename from .github/workflows/docker-dev-release.yml rename to .github/workflows/docker-image.yml index dda0554d..0368c3ac 100644 --- a/.github/workflows/docker-dev-release.yml +++ b/.github/workflows/docker-image.yml @@ -6,7 +6,8 @@ on: - development - stage - test - + release: + types: [ published ] jobs: push_to_registry: name: Push Docker branch image to Docker Hub @@ -22,24 +23,33 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: - images: agroportal/ontologies_api + images: | + agroportal/ontologies_api + ghcr.io/${{ github.repository }} - name: Build and push Docker image uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64, linux/arm64 build-args: | - RUBY_VERSION=2.7 + RUBY_VERSION=2.7.8 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 6b2c973d..d6592394 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: copy-env-config + run: cp .env.sample .env - name: Build docker-compose run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step - name: Run unit tests @@ -19,7 +21,7 @@ jobs: # http://docs.codecov.io/docs/testing-with-docker run: | ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v' + docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- (bundle check || bundle install) && bundle exec rake test TESTOPTS='-v' - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: diff --git a/.gitignore b/.gitignore index 886a220f..8b568832 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,5 @@ test/data/ontology_files/catalog-v001.xml create_permissions.log ontologies_api.iml + +.env diff --git a/Gemfile.lock b/Gemfile.lock index 8ff9e209..9875b7fc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 7d16b7b1ecc4da8bfd93db2a106737b06d84d6ff + revision: 4c7dfa80a8bb4a7d8cfb7ad1fc8a1a88e420e59e branch: development specs: ontologies_linked_data (0.0.1) @@ -109,7 +109,7 @@ GEM sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) base64 (0.2.0) - bcrypt (3.1.19) + bcrypt (3.1.20) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) @@ -134,8 +134,7 @@ GEM date (3.3.4) declarative (0.0.20) docile (1.4.0) - domain_name (0.5.20190701) - unf (>= 0.0.5, < 1.0.0) + domain_name (0.6.20231109) ed25519 (1.3.0) faraday (1.10.3) faraday-em_http (~> 1.0) @@ -217,7 +216,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.4.4) + net-imap (0.4.6) date net-protocol net-pop (0.1.2) @@ -242,7 +241,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.3) + public_suffix (5.0.4) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -321,7 +320,7 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.5) + sshkit (1.21.6) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) @@ -332,9 +331,6 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.2) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) diff --git a/README.md b/README.md index dfaa77ea..b4caa10a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,53 @@ ontologies_api provides a RESTful interface for accessing [BioPortal](https://bioportal.bioontology.org/) (an open repository of biomedical ontologies). Supported services include downloads, search, access to terms and concepts, text annotation, and much more. -## Prerequisites +# Run ontologies_api + +## Using OntoPortal api utilities script +### See help + +```bash +bin/ontoportal help +``` + +``` +Usage: bin/ontoportal {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY] + dev : Start the Ontoportal API development server. + Example: bin/ontoportal dev --api-url http://localhost:9393 + Use --reset-cache to remove volumes: bin/ontoportal dev --reset-cache + test : Run tests. + run : Run a command in the Ontoportal API Docker container. + help : Show this help message. + +Description: + This script provides convenient commands for managing an Ontoportal API + application using Docker Compose. It includes options for starting the development server, + running tests, and executing commands within the Ontoportal API Docker container. + +Goals: + - Simplify common tasks related to Ontoportal API development using Docker. + - Provide a consistent and easy-to-use interface for common actions. + + +``` +### Configuration +``` +cp .env.sample .env +``` + +### Run dev +```bash +bin/ontoportal dev +``` + +### Run test with a local OntoPortal API +```bash +bin/ontoportal test +``` + + +## Manually +### Prerequisites - [Ruby 2.x](http://www.ruby-lang.org/en/downloads/) (most recent patch level) - [rbenv](https://github.com/sstephenson/rbenv) and [ruby-build](https://github.com/sstephenson/ruby-build) (optional) @@ -19,7 +65,7 @@ ontologies_api provides a RESTful interface for accessing [BioPortal](https://bi - [Solr](http://lucene.apache.org/solr/) - BioPortal indexes ontology class and property content using Solr (a Lucene-based server) -## Configuring Solr +### Configuring Solr To configure Solr for ontologies_api usage, modify the example project included with Solr by doing the following: @@ -46,22 +92,22 @@ To configure Solr for ontologies_api usage, modify the example project included # Edit the ontologieS_api/config/environments/{env}.rb file to point to your running instance: # http://localhost:8983/solr/NCBO1 -## Installing +### Installing -### Clone the repository +#### Clone the repository ``` $ git clone git@github.com:ncbo/ontologies_api.git $ cd ontologies_api ``` -### Install the dependencies +#### Install the dependencies ``` $ bundle install ``` -### Create an environment configuration file +#### Create an environment configuration file ``` $ cp config/environments/config.rb.sample config/environments/development.rb @@ -73,7 +119,7 @@ production.rb
development.rb
test.rb -### Run the unit tests (optional) +#### Run the unit tests (optional) Requires a configuration file for the test environment: @@ -87,7 +133,7 @@ Execute the suite of tests from the command line: $ bundle exec rake test ``` -### Run the application +#### Run the application ``` $ bundle exec rackup --port 9393 diff --git a/bin/ontoportal b/bin/ontoportal new file mode 100755 index 00000000..573b49c7 --- /dev/null +++ b/bin/ontoportal @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Function to display script usage information +show_help() { + echo "Usage: $0 {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY]" + echo " dev : Start the Ontoportal API development server." + echo " Example: $0 dev --api-url http://localhost:9393" + echo " Use --reset-cache to remove volumes: $0 dev --reset-cache" + echo " test : Run tests." + echo " run : Run a command in the Ontoportal API Docker container." + echo " help : Show this help message." + echo + echo "Description:" + echo " This script provides convenient commands for managing an Ontoportal API" + echo " application using Docker Compose. It includes options for starting the development server," + echo " running tests, and executing commands within the Ontoportal API Docker container." + echo + echo "Goals:" + echo " - Simplify common tasks related to Ontoportal API development using Docker." + echo " - Provide a consistent and easy-to-use interface for common actions." +} +# Function to update or create the .env file with API_URL and API_KEY +update_env_file() { + local api_url="$1" + + # Update the .env file with the provided values + file_content=$(<.env) + + # Make changes to the variable + while IFS= read -r line; do + if [[ "$line" == "API_URL="* ]]; then + echo "API_URL=$api_url" + else + echo "$line" + fi + done <<< "$file_content" > .env +} + +# Function to create configuration files if they don't exist +create_config_files() { + if [ ! -f ".env" ]; then + echo "Creating .env file from env.sample" + cp .env.sample .env + fi + + if [ ! -f "config/environments/development.rb" ]; then + echo "Creating config/environments/development.rb file from config/environments/config.rb.sample" + cp config/bioportal_config_env.rb.sample config/bioportal_config_development.rb + fi +} + +# Function to handle the "dev" option +dev() { + echo "Starting Ontoportal API development server..." + + create_config_files + local reset_cache=false + local api_url="" + + + # Check for command line arguments + while [[ "$#" -gt 0 ]]; do + case $1 in + --reset-cache) + reset_cache=true + shift + ;; + --api-url) + api_url="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + + + # Check if arguments are provided + if [ -n "$api_url" ] ; then + # If arguments are provided, update the .env file + update_env_file "$api_url" + else + # If no arguments, fetch values from the .env file + source .env + api_url="$API_URL" + fi + + if [ -z "$api_url" ] ; then + echo "Error: Missing required arguments. Please provide both --api-url or update them in your .env" + exit 1 + fi + + # Check if --reset-cache is present and execute docker compose down --volumes + if [ "$reset_cache" = true ]; then + echo "Resetting cache. Running: docker compose down --volumes" + docker compose down --volumes + fi + + echo "Run: bundle exec api s -b 0.0.0.0 -p 3000" + docker compose run --rm -it --service-ports api bash -c "(bundle check || bundle install) && bundle exec rackup -o 0.0.0.0 --port 9393" +} + +# Function to handle the "test" option +test() { + + + local api_url="" + local test_path="" + local test_options="" + + # Check for command line arguments + while [ "$#" -gt 0 ]; do + case "$1" in + --api-url) + shift + api_url="$1" + ;; + *) + if [ -z "$test_path" ]; then + test_path="$1" + else + test_options="$test_options $1" + fi + ;; + esac + shift + done + + + + script="API_URL=$api_url bundle exec rake test TEST=\"$test_path\" TESTOPTS=\"$test_options\"" + echo "Running tests..." + echo "Run: $script" + + docker compose run --rm -it api bash -c "(bundle check || bundle install) && $script" +} + +# Function to handle the "run" option +run() { + echo "Run: $*" + docker compose run --rm -it api bash -c "$*" +} + +# Main script logic +case "$1" in + "run") + run "${@:2}" + ;; + "dev") + dev "${@:2}" + ;; + "test") + test "${@:2}" + ;; + "help") + show_help + ;; + *) + show_help + exit 1 + ;; +esac diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 45ebf2be..8713b9f2 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -18,7 +18,7 @@ REDIS_PERSISTENT_HOST = ENV.include?("REDIS_PERSISTENT_HOST") ? ENV["REDIS_PERSI REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/ontologies_report.json" REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" -REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : "http://localhost:9393" +REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : ENV["API_URL"] || "http://localhost:9393" SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" diff --git a/docker-compose.yml b/docker-compose.yml index 5cb64963..f7325381 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,5 @@ x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_api:0.0.1 + image: agroportal/ontologies_api:development environment: &env BUNDLE_PATH: /srv/ontoportal/bundle # default bundle config resolves to /usr/local/bundle/config inside of the container @@ -39,6 +34,8 @@ x-app: &app services: api: <<: *app + env_file: + .env environment: <<: *env GOO_BACKEND_NAME: 4store From be861eb6edfddde4737e332e5872265eba9642e4 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Sun, 26 Nov 2023 07:45:52 +0100 Subject: [PATCH 68/70] fix date list properties population helper --- .github/workflows/docker-image.yml | 2 +- Gemfile.lock | 2 +- helpers/application_helper.rb | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0368c3ac..1f82c680 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -47,7 +47,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64, linux/arm64 + platforms: linux/amd64 build-args: | RUBY_VERSION=2.7.8 push: true diff --git a/Gemfile.lock b/Gemfile.lock index 9875b7fc..1381b423 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 4c7dfa80a8bb4a7d8cfb7ad1fc8a1a88e420e59e + revision: a199eff007f5d7f18205d61194f3823445aa6460 branch: development specs: ontologies_linked_data (0.0.1) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 172170fa..d90630a3 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -88,7 +88,10 @@ def populate_from_params(obj, params) value = retrieved_values elsif attribute_settings && attribute_settings[:enforce] && attribute_settings[:enforce].include?(:date_time) # TODO: Remove this awful hack when obj.class.model_settings[:range][attribute] contains DateTime class - value = DateTime.parse(value) + is_array = value.is_a?(Array) + value = Array(value).map{ |v| DateTime.parse(v) } + value = value.first unless is_array + value elsif attribute_settings && attribute_settings[:enforce] && attribute_settings[:enforce].include?(:uri) && attribute_settings[:enforce].include?(:list) # in case its a list of URI, convert all value to IRI value = value.map { |v| RDF::IRI.new(v) } From 708be2c5cd7beeb3e7d60e334ac7febcdcd6a94d Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Tue, 28 Nov 2023 14:07:09 +0100 Subject: [PATCH 69/70] Feature: update ontoportal bash script to handle local gems binding (#61) * add ontoportal bash script to run test and development servers * update README.md * update docker CI to work in production releases * update ontoportal script to handle local gems bindq * update ontoportal script to handle binding to local gem for development * fixing the test runner after the new changes in the ontoportal script --- .env.sample | 5 +- .github/workflows/docker-image.yml | 2 +- README.md | 5 +- bin/ontoportal | 197 ++++++++++++++++++++--------- 4 files changed, 141 insertions(+), 68 deletions(-) diff --git a/.env.sample b/.env.sample index 719949b1..2c15a1c0 100644 --- a/.env.sample +++ b/.env.sample @@ -1 +1,4 @@ -API_URL=http://localhost:9393 \ No newline at end of file +API_URL=http://localhost:9393 +ONTOLOGIES_LINKED_DATA_PATH= +GOO_PATH= +SPARQL_CLIENT_PATH= \ No newline at end of file diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 1f82c680..737482f9 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -47,7 +47,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 build-args: | RUBY_VERSION=2.7.8 push: true diff --git a/README.md b/README.md index b4caa10a..02b9f076 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,7 @@ Goals: ``` -### Configuration -``` -cp .env.sample .env -``` + ### Run dev ```bash diff --git a/bin/ontoportal b/bin/ontoportal index 573b49c7..4840dad3 100755 --- a/bin/ontoportal +++ b/bin/ontoportal @@ -2,34 +2,61 @@ # Function to display script usage information show_help() { - echo "Usage: $0 {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY]" - echo " dev : Start the Ontoportal API development server." - echo " Example: $0 dev --api-url http://localhost:9393" - echo " Use --reset-cache to remove volumes: $0 dev --reset-cache" - echo " test : Run tests." - echo " run : Run a command in the Ontoportal API Docker container." - echo " help : Show this help message." - echo - echo "Description:" - echo " This script provides convenient commands for managing an Ontoportal API" - echo " application using Docker Compose. It includes options for starting the development server," - echo " running tests, and executing commands within the Ontoportal API Docker container." - echo - echo "Goals:" - echo " - Simplify common tasks related to Ontoportal API development using Docker." - echo " - Provide a consistent and easy-to-use interface for common actions." + cat << EOL +Usage: $0 {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY] [--old-path OLD_PATH] [--goo-path GOO_PATH] [--sparql-client-path SPARQL_CLIENT_PATH] + dev : Start the Ontoportal API development server. + Example: $0 dev --api-url http://localhost:9393 + Use --reset-cache to remove volumes: $0 dev --reset-cache + test : Run tests. Specify either a test file or use 'all'. + Example: $0 test test/controllers/test_users_controller.rb -v --name=name_of_the_test + Example (run all tests): $0 test all -v + run : Run a command in the Ontoportal API Docker container. + help : Show this help message. + +Description: + This script provides convenient commands for managing an Ontoportal API + application using Docker Compose. It includes options for starting the development server, + running tests, and executing commands within the Ontoportal API Docker container. + +Options: + --reset-cache : Remove Docker volumes (used with 'dev'). + --api-url API_URL : Specify the API URL. + --api-key API_KEY : Specify the API key. + --old-path OLD_PATH : Specify the path for ontologies_linked_data. + --goo-path GOO_PATH : Specify the path for goo. + --sparql-client-path : Specify the path for sparql-client. + test_file | all : Specify either a test file or all the tests will be run. + -v : Enable verbosity. + --name=name_of_the_test : Specify the name of the test. + +Goals: + - Simplify common tasks related to Ontoportal API development using Docker. + - Provide a consistent and easy-to-use interface for common actions. +EOL } + + # Function to update or create the .env file with API_URL and API_KEY update_env_file() { + # Update the .env file with the provided values local api_url="$1" + local old_path="$2" + local goo_path="$3" + local sparql_client_path="$4" # Update the .env file with the provided values file_content=$(<.env) # Make changes to the variable while IFS= read -r line; do - if [[ "$line" == "API_URL="* ]]; then + if [[ "$line" == "API_URL="* && -n "$api_url" ]]; then echo "API_URL=$api_url" + elif [[ "$line" == "ONTOLOGIES_LINKED_DATA_PATH="* ]]; then + echo "ONTOLOGIES_LINKED_DATA_PATH=$old_path" + elif [[ "$line" == "GOO_PATH="* ]]; then + echo "GOO_PATH=$goo_path" + elif [[ "$line" == "SPARQL_CLIENT_PATH="* ]]; then + echo "SPARQL_CLIENT_PATH=$sparql_client_path" else echo "$line" fi @@ -38,26 +65,52 @@ update_env_file() { # Function to create configuration files if they don't exist create_config_files() { - if [ ! -f ".env" ]; then - echo "Creating .env file from env.sample" - cp .env.sample .env - fi + [ -f ".env" ] || cp .env.sample .env + [ -f "config/environments/development.rb" ] || cp config/environments/config.rb.sample config/environments/development.rb +} - if [ ! -f "config/environments/development.rb" ]; then - echo "Creating config/environments/development.rb file from config/environments/config.rb.sample" - cp config/bioportal_config_env.rb.sample config/bioportal_config_development.rb - fi +# Function to build Docker run command with conditionally added bind mounts +build_docker_run_cmd() { + local custom_command="$1" + local old_path="$2" + local goo_path="$3" + local sparql_client_path="$4" + + local docker_run_cmd="docker compose run --rm -it" + local bash_cmd="" + + # Conditionally add bind mounts only if the paths are not empty + for path_var in "old_path:ontologies_linked_data" "goo_path:goo" "sparql_client_path:sparql-client"; do + IFS=':' read -r path value <<< "$path_var" + + if [ -n "${!path}" ]; then + host_path="$(realpath "$(dirname "${!path}")")/$value" + echo "Run: bundle config local.$value ${!path}" + container_path="/srv/ontoportal/$value" + docker_run_cmd+=" -v $host_path:$container_path" + bash_cmd+="(git config --global --add safe.directory $container_path && bundle config local.$value $container_path) &&" + else + bash_cmd+=" (bundle config unset local.$value) &&" + fi + done + + bash_cmd+=" (bundle check || bundle install || bundle update) && $custom_command" + docker_run_cmd+=" --service-ports api bash -c \"$bash_cmd\"" + + eval "$docker_run_cmd" } -# Function to handle the "dev" option -dev() { - echo "Starting Ontoportal API development server..." - - create_config_files +# Function to handle the "dev" and "test" options +run_command() { + local custom_command="$1" + local reset_cache=false local api_url="" + local old_path="" + local goo_path="" + local sparql_client_path="" - + shift # Check for command line arguments while [[ "$#" -gt 0 ]]; do case $1 in @@ -69,6 +122,18 @@ dev() { api_url="$2" shift 2 ;; + --old-path) + old_path="$2" + shift 2 + ;; + --goo-path) + goo_path="$2" + shift 2 + ;; + --sparql-client-path) + sparql_client_path="$2" + shift 2 + ;; *) echo "Unknown option: $1" show_help @@ -77,48 +142,58 @@ dev() { esac done - + # Check if --reset-cache is present and execute docker compose down --volumes + if [ "$reset_cache" = true ]; then + echo "Resetting cache. Running: docker compose down --volumes" + docker compose down --volumes + fi # Check if arguments are provided - if [ -n "$api_url" ] ; then - # If arguments are provided, update the .env file - update_env_file "$api_url" - else - # If no arguments, fetch values from the .env file - source .env - api_url="$API_URL" - fi + update_env_file "$api_url" "$old_path" "$goo_path" "$sparql_client_path" + + + + # If no arguments, fetch values from the .env file + source .env + api_url="$API_URL" + old_path="$ONTOLOGIES_LINKED_DATA_PATH" + goo_path="$GOO_PATH" + sparql_client_path="$SPARQL_CLIENT_PATH" + if [ -z "$api_url" ] ; then echo "Error: Missing required arguments. Please provide both --api-url or update them in your .env" exit 1 fi - # Check if --reset-cache is present and execute docker compose down --volumes - if [ "$reset_cache" = true ]; then - echo "Resetting cache. Running: docker compose down --volumes" - docker compose down --volumes - fi - echo "Run: bundle exec api s -b 0.0.0.0 -p 3000" - docker compose run --rm -it --service-ports api bash -c "(bundle check || bundle install) && bundle exec rackup -o 0.0.0.0 --port 9393" + + # Build the Docker run command + echo "Run: $custom_command" + build_docker_run_cmd "$custom_command" "$old_path" "$goo_path" "$sparql_client_path" } -# Function to handle the "test" option -test() { +# Function to handle the "dev" option +dev() { + echo "Starting OntoPortal API development server..." + local custom_command="bundle exec shotgun --host 0.0.0.0 --env=development" + run_command "$custom_command" "$@" +} - local api_url="" +# Function to handle the "test" option +test() { + echo "Running tests..." local test_path="" local test_options="" - + local all_arguments=() # Check for command line arguments while [ "$#" -gt 0 ]; do case "$1" in - --api-url) - shift - api_url="$1" - ;; + --api-url | --reset-cache | --old-path | --goo-path | --sparql-client-path) + all_arguments+=("$1" "$2") + shift 2 + ;; *) if [ -z "$test_path" ]; then test_path="$1" @@ -130,13 +205,9 @@ test() { shift done - - - script="API_URL=$api_url bundle exec rake test TEST=\"$test_path\" TESTOPTS=\"$test_options\"" - echo "Running tests..." - echo "Run: $script" - - docker compose run --rm -it api bash -c "(bundle check || bundle install) && $script" + local custom_command="bundle exec rake test TEST='$test_path' TESTOPTS='$test_options'" + echo "run : $custom_command" + run_command "$custom_command" "${all_arguments[@]}" } # Function to handle the "run" option @@ -145,6 +216,8 @@ run() { docker compose run --rm -it api bash -c "$*" } +create_config_files + # Main script logic case "$1" in "run") From 14e9a2bf62c0d0a52a56ff84018c089a60a40b00 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 30 Nov 2023 04:06:07 +0100 Subject: [PATCH 70/70] add description filter to the submissions endpoint (#62) --- helpers/request_params_helper.rb | 27 ++++++++++++------- .../test_ontology_submissions_controller.rb | 5 ++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index f1b8268c..59adeba7 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -122,17 +122,24 @@ def add_inverse_filters(inverse_filters, query) end def add_acronym_name_filters(query) - if params[:acronym] - filter = Goo::Filter.new(extract_attr(:ontology_acronym)).regex(params[:acronym]) - if params[:name] - filter.or(Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name])) - end - query = query.filter(filter) - elsif params[:name] - filter = Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name]) - query = query.filter(filter) + filters = { + acronym: :ontology_acronym, + name: :ontology_name, + description: :description + }.map do |key, attr| + (params[key].nil? || params[key].empty?) ? nil : [extract_attr(attr), params[key]] + end.compact + + return query if filters.empty? + + key, val = filters.first + filter = Goo::Filter.new(key).regex(val) + + filters.drop(1).each do |k, v| + filter = filter.or(Goo::Filter.new(k).regex(v)) end - query + + query.filter(filter) end def add_order_by_patterns(query) diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 147b9c12..095d0339 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -314,6 +314,7 @@ def test_submissions_pagination_filter ontologies.first.save sub = ontologies.first.latest_submission(status: :any).bring_remaining sub.status = 'retired' + sub.description = "234" sub.creationDate = DateTime.yesterday.to_datetime sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first sub.save @@ -363,6 +364,10 @@ def test_submissions_pagination_filter submissions = MultiJson.load(last_response.body) refute_empty submissions["collection"] assert_equal ontologies.size - 1 , submissions["collection"].size + get "/submissions?page=1&pagesize=100&description=234&acronym=234&name=234" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1 , submissions["collection"].size end def test_submissions_default_includes