diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 4dc9e323..16d8357e 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -7,25 +7,35 @@ on: jobs: test: strategy: + fail-fast: false matrix: - backend: ['api'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend + goo-slice: [ '20', '100', '500' ] + ruby-version: [ '2.7' ] + triplestore: [ 'fs', 'ag', 'vo', 'gb' ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: copy-env-config - run: cp .env.sample .env - - name: Build docker-compose - run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step - - name: Run unit tests - # unit tests are run inside a container - # http://docs.codecov.io/docs/testing-with-docker - run: | - ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle install - docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v' - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - flags: unittests - verbose: true - fail_ci_if_error: false # optional (default = false) + - uses: actions/checkout@v3 + - name: Install Dependencies + run: sudo apt-get update && sudo apt-get -y install raptor2-utils + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run unit tests + # unit tests are run inside a container + # http://docs.codecov.io/docs/testing-with-docker + run: | + ci_env=`bash <(curl -s https://codecov.io/env)` + GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v" + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + flags: unittests + verbose: true + fail_ci_if_error: false # optional (default = false) diff --git a/Dockerfile b/Dockerfile index d65d517b..0886a433 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ openjdk-11-jre-headless \ raptor2-utils \ wait-for-it \ + libraptor2-dev \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /srv/ontoportal/ontologies_api diff --git a/Gemfile b/Gemfile index e2a713f1..bd445a1e 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,9 @@ gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' gem 'request_store' +gem 'parallel' +gem 'json-ld' + # Rack middleware gem 'ffi' @@ -27,9 +30,8 @@ gem 'rack-timeout' gem 'redis-rack-cache', '~> 2.0' # Data access (caching) -gem 'redis', '~> 4.8.1' -gem 'redis-activesupport' -gem 'redis-store', '1.9.1' +gem 'redis' +gem 'redis-store', '~>1.10' # Monitoring gem 'cube-ruby', require: 'cube' @@ -44,12 +46,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'master' -gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master' +gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' -gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'master' +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development' +gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 @@ -74,5 +76,5 @@ group :test do gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io - gem 'webmock' + gem 'webmock', '~> 3.19.1' end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index acf6a02a..9984c044 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,13 +11,16 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 8ddd2d719617ad082c6964a9efdac153cdd2b48e - branch: master + revision: b2a635fb1e8206e6e3010be4dbe033b47eb58481 + branch: development specs: goo (0.0.2) addressable (~> 2.8) pry - rdf (= 1.0.8) + rdf (= 3.2.11) + rdf-raptor + rdf-rdfxml + rdf-vocab redis rest-client rsolr @@ -26,8 +29,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git - revision: 57204d8e54432ba660af4c49806e2a3019a23fa2 - branch: master + revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + branch: development specs: ncbo_annotator (0.0.1) goo @@ -54,8 +57,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 95e77989e4e8ea2fde86cf4f048f7d6cd7a6829f - branch: master + revision: 1278507e6ae8224edca746c7c150775ec2210195 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -74,13 +77,12 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/sparql-client.git - revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0 - branch: master + revision: 82302db1bfaec6593f3ea26917d7f2bb2dd485ce + branch: development specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) + sparql-client (3.2.2) + net-http-persistent (~> 4.0, >= 4.0.2) + rdf (~> 3.2, >= 3.2.11) GIT remote: https://github.com/palexander/rack-post-body-to-params.git @@ -106,15 +108,15 @@ GEM multi_json (~> 1.0) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) - airbrussh (1.5.1) + airbrussh (1.5.2) sshkit (>= 1.6.1, != 1.7.0) - backports (3.24.1) + backports (3.25.0) base64 (0.2.0) bcrypt (3.1.20) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.18.0) + capistrano (3.18.1) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -128,6 +130,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.3) + connection_pool (2.4.1) crack (1.0.0) bigdecimal rexml @@ -172,15 +175,15 @@ GEM grpc (~> 1.59) get_process_mem (0.2.7) ffi (~> 1.0) - google-analytics-data (0.5.0) + google-analytics-data (0.6.0) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.11.2) + google-analytics-data-v1beta (0.12.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) - google-apis-analytics_v3 (0.14.0) - google-apis-core (>= 0.12.0, < 2.a) - google-apis-core (0.13.0) + google-apis-analytics_v3 (0.15.0) + google-apis-core (>= 0.14.0, < 2.a) + google-apis-core (0.14.1) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.1, < 3.a) @@ -188,18 +191,19 @@ GEM representable (~> 3.0) retriable (>= 2.0, < 4.a) rexml - google-cloud-core (1.6.1) + google-cloud-core (1.7.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) - google-cloud-errors (1.3.1) + google-cloud-errors (1.4.0) + google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) - googleapis-common-protos (1.4.0) - google-protobuf (~> 3.14) - googleapis-common-protos-types (~> 1.2) - grpc (~> 1.27) - googleapis-common-protos-types (1.12.0) + googleapis-common-protos (1.5.0) + google-protobuf (~> 3.18) + googleapis-common-protos-types (~> 1.7) + grpc (~> 1.41) + googleapis-common-protos-types (1.14.0) google-protobuf (~> 3.18) googleauth (1.11.0) faraday (>= 1.0, < 3.a) @@ -208,27 +212,34 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.61.0-x86_64-linux) + grpc (1.62.0-x86_64-darwin) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) + grpc (1.62.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt hashdiff (1.1.0) + htmlentities (4.3.4) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) + json (2.7.2) + json-ld (3.0.2) + multi_json (~> 1.12) + rdf (>= 2.2.8, < 4.0) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.7.1) - jwt (2.8.0) + jwt (2.8.1) base64 kgio (2.11.4) - libxml-ruby (5.0.2) + libxml-ruby (5.0.3) + link_header (0.0.8) logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) @@ -237,10 +248,10 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0206) + mime-types-data (3.2024.0305) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -249,7 +260,8 @@ GEM multi_json (1.15.0) multipart-post (2.4.0) mutex_m (0.2.0) - net-http-persistent (2.9.4) + net-http-persistent (4.0.2) + connection_pool (~> 2.2) net-imap (0.4.10) date net-protocol @@ -261,22 +273,23 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-smtp (0.4.0.1) + net-smtp (0.5.0) net-protocol - net-ssh (7.2.1) + net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.7.1) + newrelic_rpm (9.9.0) oj (2.18.5) omni_logger (0.1.4) logger os (1.1.4) + parallel (1.24.0) parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) + public_suffix (5.0.5) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -295,18 +308,31 @@ GEM rack-timeout (0.6.3) raindrops (0.20.1) rake (10.5.0) - rdf (1.0.8) - addressable (>= 2.2) + rdf (3.2.11) + link_header (~> 0.0, >= 0.0.8) + rdf-raptor (3.2.0) + ffi (~> 1.15) + rdf (~> 3.2) + rdf-rdfxml (3.2.2) + builder (~> 3.2) + htmlentities (~> 4.3) + rdf (~> 3.2) + rdf-xsd (~> 3.2) + rdf-vocab (3.2.7) + rdf (~> 3.2, >= 3.2.4) + rdf-xsd (3.2.1) + rdf (~> 3.2) + rexml (~> 3.2) redcarpet (3.6.0) - redis (4.8.1) - redis-activesupport (5.3.0) - activesupport (>= 3, < 8) - redis-store (>= 1.3, < 2) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) + connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) + redis-store (1.10.0) + redis (>= 4, < 6) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -320,7 +346,7 @@ GEM netrc (~> 0.8) retriable (3.1.2) rexml (3.2.6) - rsolr (2.5.0) + rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) ruby-xxHash (0.4.0.2) @@ -355,7 +381,8 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.0) + sshkit (1.22.2) + base64 mutex_m net-scp (>= 1.1.2) net-sftp (>= 2.1.2) @@ -376,12 +403,13 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) - webmock (3.22.0) + webmock (3.19.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS + x86_64-darwin-23 x86_64-linux DEPENDENCIES @@ -398,6 +426,7 @@ DEPENDENCIES ffi goo! haml (~> 5.2.2) + json-ld json-schema (~> 2.0) minitest (~> 4.0) minitest-stub_any_instance @@ -408,6 +437,7 @@ DEPENDENCIES newrelic_rpm oj (~> 2.0) ontologies_linked_data! + parallel parseconfig pry rack @@ -421,10 +451,9 @@ DEPENDENCIES rack-timeout rake (~> 10.0) redcarpet - redis (~> 4.8.1) - redis-activesupport + redis redis-rack-cache (~> 2.0) - redis-store (= 1.9.1) + redis-store (~> 1.10) request_store shotgun! simplecov @@ -435,7 +464,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer - webmock + webmock (~> 3.19.1) BUNDLED WITH - 2.3.23 + 2.4.22 diff --git a/bin/ontoportal b/bin/ontoportal index 4840dad3..66f1a654 100755 --- a/bin/ontoportal +++ b/bin/ontoportal @@ -76,7 +76,7 @@ build_docker_run_cmd() { local goo_path="$3" local sparql_client_path="$4" - local docker_run_cmd="docker compose run --rm -it" + local docker_run_cmd="docker compose -p ontoportal_docker run --rm -it --name api-service" local bash_cmd="" # Conditionally add bind mounts only if the paths are not empty @@ -100,6 +100,7 @@ build_docker_run_cmd() { eval "$docker_run_cmd" } + # Function to handle the "dev" and "test" options run_command() { local custom_command="$1" @@ -177,7 +178,7 @@ run_command() { dev() { echo "Starting OntoPortal API development server..." - local custom_command="bundle exec shotgun --host 0.0.0.0 --env=development" + local custom_command="bundle exec shotgun --host 0.0.0.0 --env=development --port 9393" run_command "$custom_command" "$@" } diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 8713b9f2..4de42655 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -48,6 +48,7 @@ LinkedData.config do |config| config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s config.replace_url_prefix = true config.rest_url_prefix = REST_URL_PREFIX.to_s + config.sparql_endpoint_url = "http://sparql.bioontology.org" # config.enable_notifications = false config.interportal_hash = { diff --git a/config/environments/test.rb b/config/environments/test.rb index 16bf407a..0b68cc3b 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -8,16 +8,18 @@ GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" REDIS_HOST = ENV.include?("REDIS_HOST") ? ENV["REDIS_HOST"] : "localhost" REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 -SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" -SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr" MGREP_HOST = ENV.include?("MGREP_HOST") ? ENV["MGREP_HOST"] : "localhost" -MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55555 +MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55556 +GOO_SLICES = ENV["GOO_SLICES"] || 500 begin # For prefLabel extract main_lang first, or anything if no main found. # For other properties only properties with a lang that is included in main_lang are used Goo.main_languages = ['en'] Goo.use_cache = false + Goo.slice_loading_size = GOO_SLICES.to_i rescue NoMethodError puts "(CNFG) >> Goo.main_lang not available" end @@ -37,6 +39,7 @@ config.ontology_analytics_redis_port = REDIS_PORT.to_i config.search_server_url = SOLR_TERM_SEARCH_URL.to_s config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s + config.sparql_endpoint_url = "http://sparql.bioontology.org" # config.enable_notifications = false config.interportal_hash = { "agroportal" => { diff --git a/config/rack_attack.rb b/config/rack_attack.rb index 60d2e3de..88a3e8d6 100644 --- a/config/rack_attack.rb +++ b/config/rack_attack.rb @@ -3,15 +3,14 @@ puts "(API) >> Throttling enabled at #{LinkedData::OntologiesAPI.settings.req_per_second_per_ip} req/sec" require 'rack/attack' -require 'redis-activesupport' use Rack::Attack attack_redis_host_port = { host: LinkedData::OntologiesAPI.settings.http_redis_host, - port: LinkedData::OntologiesAPI.settings.http_redis_port + port: LinkedData::OntologiesAPI.settings.http_redis_port, + db: 1 } -attack_store = ActiveSupport::Cache::RedisStore.new(attack_redis_host_port) -Rack::Attack.cache.store = attack_store +Rack::Attack.cache.store = Redis.new(attack_redis_host_port) safe_ips = LinkedData::OntologiesAPI.settings.safe_ips ||= Set.new safe_ips.each do |safe_ip| diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml deleted file mode 100644 index 72e7b7d3..00000000 --- a/config/solr/property_search/enumsconfig.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - ONTOLOGY - VALUE_SET_COLLECTION - - - ANNOTATION - DATATYPE - OBJECT - - \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede77425..00000000 --- a/config/solr/property_search/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# Á => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Å => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# Ì => I -"\u00CC" => "I" - -# Í => I -"\u00CD" => "I" - -# Î => I -"\u00CE" => "I" - -# Ï => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# Ð => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ò => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Œ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ü => U -"\u00DC" => "U" - -# Ý => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# å => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# œ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# fi => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml deleted file mode 100644 index 20824ea6..00000000 --- a/config/solr/property_search/schema.xml +++ /dev/null @@ -1,1179 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml deleted file mode 100644 index 771a0f32..00000000 --- a/config/solr/property_search/solrconfig.xml +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - 8.8.2 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - ${solr.max.booleanClauses:500000} - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - - - - - - - - - explicit - json - true - - - - - - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - false - - - terms - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z - yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z - yyyy-MM-dd HH:mm[:ss[.SSS]][z - yyyy-MM-dd HH:mm[:ss[,SSS]][z - [EEE, ]dd MMM yyyy HH:mm[:ss] z - EEEE, dd-MMM-yy HH:mm:ss z - EEE MMM ppd HH:mm:ss [z ]yyyy - - - - - java.lang.String - text_general - - *_str - 256 - - - true - - - java.lang.Boolean - booleans - - - java.util.Date - pdates - - - java.lang.Long - java.lang.Integer - plongs - - - java.lang.Number - pdoubles - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - - - - - - - - - - diff --git a/config/solr/solr.xml b/config/solr/solr.xml deleted file mode 100644 index d9d089e4..00000000 --- a/config/solr/solr.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - ${solr.max.booleanClauses:500000} - ${solr.sharedLib:} - ${solr.allowPaths:} - - - - ${host:} - ${solr.port.advertise:0} - ${hostContext:solr} - - ${genericCoreNodeNames:true} - - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - - - ${socketTimeout:600000} - ${connTimeout:60000} - ${solr.shardsWhitelist:} - - - - - diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml deleted file mode 100644 index 72e7b7d3..00000000 --- a/config/solr/term_search/enumsconfig.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - ONTOLOGY - VALUE_SET_COLLECTION - - - ANNOTATION - DATATYPE - OBJECT - - \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede77425..00000000 --- a/config/solr/term_search/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# Á => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Å => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# Ì => I -"\u00CC" => "I" - -# Í => I -"\u00CD" => "I" - -# Î => I -"\u00CE" => "I" - -# Ï => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# Ð => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ò => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Œ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ü => U -"\u00DC" => "U" - -# Ý => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# å => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# œ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# fi => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml deleted file mode 100644 index fa95e127..00000000 --- a/config/solr/term_search/schema.xml +++ /dev/null @@ -1,1222 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml deleted file mode 100644 index 771a0f32..00000000 --- a/config/solr/term_search/solrconfig.xml +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - 8.8.2 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - ${solr.max.booleanClauses:500000} - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - - - - - - - - - explicit - json - true - - - - - - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - false - - - terms - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z - yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z - yyyy-MM-dd HH:mm[:ss[.SSS]][z - yyyy-MM-dd HH:mm[:ss[,SSS]][z - [EEE, ]dd MMM yyyy HH:mm[:ss] z - EEEE, dd-MMM-yy HH:mm:ss z - EEE MMM ppd HH:mm:ss [z ]yyyy - - - - - java.lang.String - text_general - - *_str - 256 - - - true - - - java.lang.Boolean - booleans - - - java.util.Date - pdates - - - java.lang.Long - java.lang.Integer - plongs - - - java.lang.Number - pdoubles - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - - - - - - - - - - diff --git a/controllers/admin_controller.rb b/controllers/admin_controller.rb index 747def93..70b94411 100644 --- a/controllers/admin_controller.rb +++ b/controllers/admin_controller.rb @@ -127,6 +127,79 @@ class AdminController < ApplicationController halt 204 end + namespace "/search" do + get '/collections' do + conn = SOLR::SolrConnector.new(Goo.search_conf, '') + collections = { collections: conn.fetch_all_collections} + reply(200, collections) + end + + get '/collections/:collection/schema' do + collection = params[:collection].to_sym + conn = SOLR::SolrConnector.new(Goo.search_conf, collection) + collection_schema = conn.fetch_schema + + reply(200, collection_schema) + end + + post '/collections/:collection/schema/init' do + collection = params[:collection].to_sym + conn = SOLR::SolrConnector.new(Goo.search_conf, collection) + collection_schema = conn.init_schema + reply(200, collection_schema) + end + + + post '/collections/:collection/search' do + collection = params[:collection].to_sym + + search_keys = %w[defType fq qf sort start rows fl stopwords lowercaseOperators] + + search_params = params.select { |key, _| search_keys.include?(key) } + search_query = params[:query] || params[:q] + search_query = search_query.blank? ? '*' : search_query + conn = SOLR::SolrConnector.new(Goo.search_conf, collection) + reply(200, conn.search(search_query, search_params).to_h) + end + + post '/index_batch/:model_name' do + error 500, "model_name parameter not set" if params["model_name"].blank? + + model = Goo.model_by_name(params["model_name"].to_sym) + error 500, "#{params["model_name"]} is not indexable" if model.nil? || !model.index_enabled? + + all_attrs = get_attributes_to_include([:all], model) + + collections = model.where.include(all_attrs).all + indexed = [] + not_indexed = [] + collections.each do |m| + begin + response = m.index.dig("responseHeader", "status") + if response.eql?(0) + indexed << m.id + else + not_indexed << m.id + end + rescue StandardError + not_indexed << m.id + end + end + + if !indexed.empty? + msg = "Batch indexing for #{model.model_name} completed for" + + if not_indexed.empty? + msg += " all models" + else + msg += " #{indexed.join(', ')} and not for the following #{not_indexed.join(', ')}, see logs for more details" + end + reply(200, msg) + else + reply(500, "Batch indexing for #{model.model_name} failed") + end + end + end private def process_long_operation(timeout, args) diff --git a/controllers/dereference_resource_controller.rb b/controllers/dereference_resource_controller.rb new file mode 100644 index 00000000..8b69efdb --- /dev/null +++ b/controllers/dereference_resource_controller.rb @@ -0,0 +1,53 @@ +use Rack::ContentNegotiation + +class DereferenceResourceController < ApplicationController + namespace "/ontologies" do + get "/:acronym/resolve/:uri" do + acronym = params[:acronym] + uri = params[:uri] + + if acronym.blank? || uri.blank? + error 500, "Usage: ontologies/:acronym/resolve/:uri?output_format= OR POST: acronym, uri, output_format parameters" + end + + output_format = env["format"].presence || params[:output_format].presence || 'application/n-triples' + + process_request(acronym, uri, output_format) + end + + private + + def process_request(acronym_param, uri_param, output_format) + acronym = acronym_param + uri = URI.decode_www_form_component(uri_param) + + error 500, "INVALID URI" unless valid_url?(uri) + sub = LinkedData::Models::Ontology.find(acronym).first&.latest_submission + + error 500, "Ontology not found" unless sub + + r = Resource.new(sub.id, uri) + case output_format + when 'application/ld+json', 'application/json' + r.to_json + when 'application/rdf+xml', 'application/xml' + r.to_xml + when 'text/turtle' + r.to_turtle + when 'application/n-triples' + r.to_ntriples + else + error 500, "Invalid output format, valid format are: application/json, application/ld+json, application/xml, application/rdf+xml, text/turtle and application/n-triples" + end + + + end + + def valid_url?(url) + uri = URI.parse(url) + uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS) + rescue URI::InvalidURIError + false + end + end +end \ No newline at end of file diff --git a/controllers/mappings_controller.rb b/controllers/mappings_controller.rb index 75f0c5b8..82c280fa 100644 --- a/controllers/mappings_controller.rb +++ b/controllers/mappings_controller.rb @@ -191,6 +191,8 @@ class MappingsController < ApplicationController .each do |m| persistent_counts[m.ontologies.first] = m.count end + ont_acronyms = restricted_ontologies_to_acronyms(params) + persistent_counts = persistent_counts.select { |key, _| ont_acronyms.include?(key) || key.start_with?("http://") } reply persistent_counts end diff --git a/controllers/properties_search_controller.rb b/controllers/properties_search_controller.rb index 29d6b772..6c5b6cdf 100644 --- a/controllers/properties_search_controller.rb +++ b/controllers/properties_search_controller.rb @@ -22,7 +22,7 @@ def process_search(params=nil) # puts "Properties query: #{query}, params: #{params}" set_page_params(params) docs = Array.new - resp = LinkedData::Models::Class.search(query, params, :property) + resp = LinkedData::Models::OntologyProperty.search(query, params) total_found = resp["response"]["numFound"] add_matched_fields(resp, Sinatra::Helpers::SearchHelper::MATCH_TYPE_LABEL) ontology_rank = LinkedData::Models::Ontology.rank diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 3bc1c13f..9f701714 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -5,16 +5,202 @@ class SearchController < ApplicationController namespace "/search" do # execute a search query get do - process_search() + process_search end post do - process_search() + process_search + end + + namespace "/ontologies" do + get do + query = params[:query] || params[:q] + groups = params.fetch("groups", "").split(',') + categories = params.fetch("hasDomain", "").split(',') + languages = params.fetch("languages", "").split(',') + status = params.fetch("status", "").split(',') + format = params.fetch("hasOntologyLanguage", "").split(',') + is_of_type = params.fetch("isOfType", "").split(',') + has_format = params.fetch("hasFormat", "").split(',') + visibility = params["visibility"] + show_views = params["show_views"] == 'true' + sort = params.fetch("sort", "score desc, ontology_name_sort asc, ontology_acronym_sort asc") + page, page_size = page_params + + fq = [ + 'resource_model:"ontology_submission"', + 'submissionStatus_txt:ERROR_* OR submissionStatus_txt:"RDF" OR submissionStatus_txt:"UPLOADED"', + groups.map { |x| "ontology_group_txt:\"http://data.bioontology.org/groups/#{x.upcase}\"" }.join(' OR '), + categories.map { |x| "ontology_hasDomain_txt:\"http://data.bioontology.org/categories/#{x.upcase}\"" }.join(' OR '), + languages.map { |x| "naturalLanguage_txt:\"#{x.downcase}\"" }.join(' OR '), + ] + + fq << "ontology_viewingRestriction_t:#{visibility}" unless visibility.blank? + fq << "!ontology_viewOf_t:*" unless show_views + + fq << format.map { |x| "hasOntologyLanguage_t:\"http://data.bioontology.org/ontology_formats/#{x}\"" }.join(' OR ') unless format.blank? + + fq << status.map { |x| "status_t:#{x}" }.join(' OR ') unless status.blank? + fq << is_of_type.map { |x| "isOfType_t:#{x}" }.join(' OR ') unless is_of_type.blank? + fq << has_format.map { |x| "hasFormalityLevel_t:#{x}" }.join(' OR ') unless has_format.blank? + + fq.reject!(&:blank?) + + if params[:qf] + qf = params[:qf] + else + qf = [ + "ontology_acronymSuggestEdge^25 ontology_nameSuggestEdge^15 descriptionSuggestEdge^10 ", # start of the word first + "ontology_acronym_text^15 ontology_name_text^10 description_text^5 ", # full word match + "ontology_acronymSuggestNgram^2 ontology_nameSuggestNgram^1.5 descriptionSuggestNgram" # substring match last + ].join(' ') + end + + page_data = search(Ontology, query, { + fq: fq, + qf: qf, + page: page, + page_size: page_size, + sort: sort + }) + + total_found = page_data.aggregate + ontology_rank = LinkedData::Models::Ontology.rank + docs = {} + acronyms_ids = {} + page_data.each do |doc| + resource_id = doc["resource_id"] + id = doc["submissionId_i"] + acronym = doc["ontology_acronym_text"] + old_resource_id = acronyms_ids[acronym] + old_id = old_resource_id.split('/').last.to_i rescue 0 + + already_found = (old_id && id && (id <= old_id)) + not_restricted = (doc["ontology_viewingRestriction_t"]&.eql?('public') || current_user&.admin?) + user_not_restricted = not_restricted || + Array(doc["ontology_viewingRestriction_txt"]).any? {|u| u.split(' ').last == current_user&.username} || + Array(doc["ontology_acl_txt"]).any? {|u| u.split(' ').last == current_user&.username} + + user_restricted = !user_not_restricted + + if acronym.blank? || already_found || user_restricted + total_found -= 1 + next + end + + docs.delete(old_resource_id) + acronyms_ids[acronym] = resource_id + + doc["ontology_rank"] = ontology_rank.dig(doc["ontology_acronym_text"], :normalizedScore) || 0.0 + docs[resource_id] = doc + end + + docs = docs.values + + docs.sort! { |a, b| [b["score"], b["ontology_rank"]] <=> [a["score"], a["ontology_rank"]] } unless params[:sort].present? + + page = page_object(docs, total_found) + + reply 200, page + end + + get '/content' do + query = params[:query] || params[:q] + page, page_size = page_params + + ontologies = params.fetch("ontologies", "").split(',') + + unless current_user&.admin? + restricted_acronyms = restricted_ontologies_to_acronyms(params) + ontologies = ontologies.empty? ? restricted_acronyms : ontologies & restricted_acronyms + end + + + types = params.fetch("types", "").split(',') + qf = params.fetch("qf", "") + + qf = [ + "ontology_t^100 resource_id^10", + "http___www.w3.org_2004_02_skos_core_prefLabel_txt^30", + "http___www.w3.org_2004_02_skos_core_prefLabel_t^30", + "http___www.w3.org_2000_01_rdf-schema_label_txt^30", + "http___www.w3.org_2000_01_rdf-schema_label_t^30", + ].join(' ') if qf.blank? + + fq = [] + + fq << ontologies.map { |x| "ontology_t:\"#{x}\"" }.join(' OR ') unless ontologies.blank? + fq << types.map { |x| "type_t:\"#{x}\" OR type_txt:\"#{x}\"" }.join(' OR ') unless types.blank? + + + conn = SOLR::SolrConnector.new(Goo.search_conf, :ontology_data) + resp = conn.search(query, fq: fq, qf: qf, defType: "edismax", + start: (page - 1) * page_size, rows: page_size) + + total_found = resp["response"]["numFound"] + docs = resp["response"]["docs"] + + + reply 200, page_object(docs, total_found) + end + end + + namespace "/agents" do + get do + query = params[:query] || params[:q] + page, page_size = page_params + type = params[:agentType].blank? ? nil : params[:agentType] + + fq = "agentType_t:#{type}" if type + + qf = [ + "acronymSuggestEdge^25 nameSuggestEdge^15 emailSuggestEdge^15 identifiersSuggestEdge^10 ", # start of the word first + "identifiers_texts^20 acronym_text^15 name_text^10 email_text^10 ", # full word match + "acronymSuggestNgram^2 nameSuggestNgram^1.5 email_text^1" # substring match last + ].join(' ') + + if params[:sort] + sort = "#{params[:sort]} asc, score desc" + else + sort = "score desc, acronym_sort asc, name_sort asc" + end + + reply 200, search(LinkedData::Models::Agent, + query, + fq: fq, qf: qf, + page: page, page_size: page_size, + sort: sort) + end end private - def process_search(params=nil) + def search(model, query, params = {}) + query = query.blank? ? "*" : query + + resp = model.search(query, search_params(params)) + + total_found = resp["response"]["numFound"] + docs = resp["response"]["docs"] + + page_object(docs, total_found) + end + + def search_params(defType: "edismax", fq:, qf:, stopwords: "true", lowercaseOperators: "true", page:, page_size:, fl: '*,score', sort:) + { + defType: defType, + fq: fq, + qf: qf, + sort: sort, + start: (page - 1) * page_size, + rows: page_size, + fl: fl, + stopwords: stopwords, + lowercaseOperators: lowercaseOperators, + } + end + + def process_search(params = nil) params ||= @params text = params["q"] @@ -50,13 +236,13 @@ def process_search(params=nil) unless params['sort'] if !text.nil? && text[-1] == '*' - docs.sort! {|a, b| [b[:score], a[:prefLabelExact].downcase, b[:ontology_rank]] <=> [a[:score], b[:prefLabelExact].downcase, a[:ontology_rank]]} + docs.sort! { |a, b| [b[:score], a[:prefLabelExact].downcase, b[:ontology_rank]] <=> [a[:score], b[:prefLabelExact].downcase, a[:ontology_rank]] } else - docs.sort! {|a, b| [b[:score], b[:ontology_rank]] <=> [a[:score], a[:ontology_rank]]} + docs.sort! { |a, b| [b[:score], b[:ontology_rank]] <=> [a[:score], a[:ontology_rank]] } end end - #need to return a Page object + # need to return a Page object page = page_object(docs, total_found) reply 200, page diff --git a/controllers/slices_controller.rb b/controllers/slices_controller.rb index a31f799e..9033222c 100644 --- a/controllers/slices_controller.rb +++ b/controllers/slices_controller.rb @@ -41,17 +41,20 @@ class SlicesController < ApplicationController ## # Create a new slice post do + error 403, "Access denied" unless current_user && current_user.admin? create_slice end # Delete a slice delete '/:slice' do + error 403, "Access denied" unless current_user && current_user.admin? LinkedData::Models::Slice.find(params[:slice]).first.delete halt 204 end # Update an existing slice patch '/:slice' do + error 403, "Access denied" unless current_user && current_user.admin? slice = LinkedData::Models::Slice.find(params[:slice]).include(LinkedData::Models::Slice.attributes(:all)).first populate_from_params(slice, params) if slice.valid? @@ -61,7 +64,7 @@ class SlicesController < ApplicationController end halt 204 end - + private def create_slice diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index 09a1835b..58e7667f 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -80,6 +80,7 @@ class UsersController < ApplicationController # Update an existing submission of an user patch '/:username' do user = User.find(params[:username]).include(User.attributes).first + params.delete("role") unless current_user.admin? populate_from_params(user, params) if user.valid? user.save @@ -98,13 +99,14 @@ class UsersController < ApplicationController private - def create_user + def create_user(send_notifications: true) params ||= @params user = User.find(params["username"]).first error 409, "User with username `#{params["username"]}` already exists" unless user.nil? + params.delete("role") unless current_user.admin? user = instance_from_params(User, params) if user.valid? - user.save(send_notifications: false) + user.save(send_notifications: send_notifications) else error 422, user.errors end diff --git a/docker-compose.yml b/docker-compose.yml index f7325381..370615a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,31 @@ x-app: &app - image: agroportal/ontologies_api:development - environment: &env - BUNDLE_PATH: /srv/ontoportal/bundle - # default bundle config resolves to /usr/local/bundle/config inside of the container - # we are setting it to local app directory if we need to use 'bundle config local' - BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle - COVERAGE: 'true' - GOO_REDIS_HOST: redis-ut - REDIS_HOST: redis-ut - REDIS_PORT: 6379 - SOLR_HOST: solr-ut - SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 - SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 - MGREP_HOST: mgrep-ut - MGREP_PORT: 55555 - stdin_open: true - tty: true - command: "bundle exec rackup -o 0.0.0.0 --port 9393" - ports: - - 9393:9393 - volumes: - # bundle volume for hosting gems installed by bundle; it helps in local development with gem udpates - - bundle:/srv/ontoportal/bundle - # api code - - .:/srv/ontoportal/ontologies_api - # mount directory containing development version of the gems if you need to use 'bundle config local' - #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal - depends_on: - - solr-ut - - redis-ut - - mgrep-ut + image: agroportal/ontologies_api:development + environment: &env + # default bundle config resolves to /usr/local/bundle/config inside of the container + # we are setting it to local app directory if we need to use 'bundle config local' + BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle + BUNDLE_PATH: /srv/ontoportal/bundle + COVERAGE: 'true' # enable simplecov code coverage + REDIS_HOST: redis-ut + REDIS_PORT: 6379 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr + stdin_open: true + tty: true + command: /bin/bash + volumes: + # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development + - bundle:/srv/ontoportal/bundle + - .:/srv/ontoportal/ontologies_api + # mount directory containing development version of the gems if you need to use 'bundle config local' + #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal + depends_on: &depends_on + solr-prop-ut: + condition: service_healthy + solr-term-ut: + condition: service_healthy + redis-ut: + condition: service_healthy services: api: @@ -51,76 +47,112 @@ services: - redis-ut - mgrep-ut - 4store-ut + ports: + - "9393:9393" - api-agraph: - <<: *app - environment: - <<: *env - GOO_BACKEND_NAME: ag - GOO_PORT: 10035 - GOO_HOST: agraph-ut - GOO_PATH_QUERY: /repositories/bioportal_test - GOO_PATH_DATA: /repositories/bioportal_test/statements - GOO_PATH_UPDATE: /repositories/bioportal_test/statements - profiles: - - agraph - depends_on: - - solr-ut - - redis-ut - - mgrep-ut - - agraph-ut + mgrep-ut: + image: ontoportal/mgrep-ncbo:0.1 + ports: + - "55556:55555" redis-ut: image: redis ports: - - 6379:6379 + - "6379:6379" + command: [ "redis-server", "--save", "", "--appendonly", "no" ] + healthcheck: + test: redis-cli ping + interval: 10s + timeout: 3s + retries: 10 4store-ut: image: bde2020/4store - #volume: fourstore:/var/lib/4store - ports: - - 9000:9000 + volumes: + - 4store:/var/lib/4store command: > - bash -c "4s-backend-setup --segments 4 ontoportal_kb - && 4s-backend ontoportal_kb - && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" + bash -c "if [ ! -d '/var/lib/4store/ontoportal_kb' ]; then 4s-backend-setup --segments 4 ontoportal_kb; fi ; 4s-backend ontoportal_kb ; 4s-httpd -D -s-1 -p 9000 ontoportal_kb" + + ports: + - "9000:9000" profiles: + - fs - 4store - solr-ut: image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro ports: - - "8983:8983" - command: > - bash -c "precreate-core term_search_core1 /configsets/term_search - && precreate-core prop_search_core1 /configsets/property_search - && solr-foreground" - - mgrep-ut: - image: ontoportal/mgrep-ncbo:0.1 - ports: - - "55556:55555" - + - 8983:8983 + command: bin/solr start -cloud -f + # volumes: + #- solr_data:/var/solr/data agraph-ut: - image: franzinc/agraph:v7.3.0 + image: franzinc/agraph:v8.1.0 + platform: linux/amd64 environment: - AGRAPH_SUPER_USER=test - AGRAPH_SUPER_PASSWORD=xyzzy shm_size: 1g - # ports: - # - 10035:10035 + ports: + # - 10035:10035 + - 10000-10035:10000-10035 + volumes: + - agdata:/agraph/data + # - ./agraph/etc:/agraph/etc command: > - bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start - ; agtool repos create bioportal_test - ; agtool users add anonymous - ; agtool users grant anonymous root:bioportal_test:rw - ; tail -f /agraph/data/agraph.log" + bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start + ; agtool repos create ontoportal_test --supersede + ; agtool users add anonymous + ; agtool users grant anonymous root:ontoportal_test:rw + ; tail -f /agraph/data/agraph.log" + # healthcheck: + # test: ["CMD-SHELL", "curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"] + # start_period: 10s + # interval: 10s + # timeout: 5s + # retries: 5 + profiles: + - ag + + virtuoso-ut: + image: tenforce/virtuoso:virtuoso7.2.5 + platform: linux/amd64 + environment: + - SPARQL_UPDATE=true + ports: + - 1111:1111 + - 8890:8890 + profiles: + - vo + healthcheck: + test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ] + start_period: 10s + interval: 60s + timeout: 5s + retries: 3 + + graphdb-ut: + image: ontotext/graphdb:10.3.3 + platform: linux/amd64 + privileged: true + environment: + GDB_HEAP_SIZE: 5G + GDB_JAVA_OPTS: >- + -Xms5g -Xmx5g + ports: + - 7200:7200 + - 7300:7300 + volumes: + - ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl + - ./test/data/graphdb-test-load.nt:/opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt + + entrypoint: > + bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb -Ddefault.min.distinct.threshold=3000 " profiles: - - agraph + - gb volumes: bundle: - #fourstore: + agdata: + 4store: + #solr_data: \ No newline at end of file diff --git a/helpers/properties_search_helper.rb b/helpers/properties_search_helper.rb index c3567edd..c4295749 100644 --- a/helpers/properties_search_helper.rb +++ b/helpers/properties_search_helper.rb @@ -33,7 +33,7 @@ def get_properties_search_query(text, params) params["qf"] = "resource_id^20 labelExact^10 labelGeneratedExact^8" params["hl.fl"] = "resource_id labelExact labelGeneratedExact" else - params["qf"] = "labelExact^100 labelGeneratedExact^80 labelSuggestEdge^50 labelSuggestNgram label labelGenerated resource_id" + params["qf"] = "labelExact^100 labelGeneratedExact^80 labelSuggestEdge^50 labelGeneratedSuggestEdge^40 labelGenerated resource_id" query = solr_escape(text) # double quote the query if it is a URL (ID searches) query = "\"#{query}\"" if text =~ /\A#{URI::regexp(['http', 'https'])}\z/ diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 071000d9..276ea3e0 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -101,15 +101,15 @@ def get_term_search_query(text, params={}) if params[EXACT_MATCH_PARAM] == "true" query = "\"#{solr_escape(text)}\"" - params["qf"] = "resource_id^20 prefLabelExact#{lang_suffix }^10 synonymExact#{lang_suffix } #{QUERYLESS_FIELDS_STR}" - params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix } synonymExact#{lang_suffix } #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^20 prefLabel#{lang_suffix}^10 synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR}" elsif params[SUGGEST_PARAM] == "true" || text[-1] == '*' text.gsub!(/\*+$/, '') query = "\"#{solr_escape(text)}\"" params["qt"] = "/suggest_ncbo" - params["qf"] = "prefLabelExact#{lang_suffix }^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["qf"] = " prefLabelExact#{lang_suffix}^100 prefLabelSuggestEdge#{lang_suffix}^50 synonym#{lang_suffix}SuggestEdge^10 prefLabel#{lang_suffix}SuggestNgram synonym#{lang_suffix}SuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" params["pf"] = "prefLabelSuggest^50" - params["hl.fl"] = "prefLabelExact#{lang_suffix } prefLabelSuggestEdge synonymSuggestEdge prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "prefLabelExact#{lang_suffix} prefLabelSuggestEdge#{lang_suffix} synonymSuggestEdge#{lang_suffix} prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" else if text.strip.empty? query = '*' @@ -117,9 +117,9 @@ def get_term_search_query(text, params={}) query = solr_escape(text) end - params["qf"] = "resource_id^100 prefLabelExact#{lang_suffix }^90 prefLabel#{lang_suffix }^70 synonymExact#{lang_suffix }^50 synonym#{lang_suffix }^10 #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^100 prefLabelExact#{lang_suffix}^90 prefLabel#{lang_suffix}^70 synonymExact#{lang_suffix}^50 synonym#{lang_suffix }^10 #{QUERYLESS_FIELDS_STR}" params["qf"] << " property" if params[INCLUDE_PROPERTIES_PARAM] == "true" - params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix } prefLabel#{lang_suffix } synonymExact#{lang_suffix } synonym#{lang_suffix } #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} prefLabel#{lang_suffix } synonymExact#{lang_suffix} synonym#{lang_suffix } #{QUERYLESS_FIELDS_STR}" params["hl.fl"] = "#{params["hl.fl"]} property" if params[INCLUDE_PROPERTIES_PARAM] == "true" end diff --git a/lib/rack/content_negotiation.rb b/lib/rack/content_negotiation.rb new file mode 100644 index 00000000..4c91da6a --- /dev/null +++ b/lib/rack/content_negotiation.rb @@ -0,0 +1,131 @@ +module Rack + class ContentNegotiation + DEFAULT_CONTENT_TYPE = "application/n-triples" # N-Triples + VARY = { 'Vary' => 'Accept' }.freeze + ENDPOINTS_FILTER = %r{^/ontologies/[^/]+/resolve/[^/]+$} # Accepted API endpoints to apply content negotiation + + # @return [#call] + attr_reader :app + + # @return [Hash{Symbol => String}] + attr_reader :options + + ## + # @param [#call] app + # @param [Hash{Symbol => Object}] options + # Other options passed to writer. + # @option options [String] :default (DEFAULT_CONTENT_TYPE) Specific content type + # @option options [RDF::Format, #to_sym] :format Specific RDF writer format to use + def initialize(app, options = {}) + @app, @options = app, options + @options[:default] = (@options[:default] || DEFAULT_CONTENT_TYPE).to_s + end + + ## + # Handles a Rack protocol request. + # Parses Accept header to find appropriate mime-type and sets content_type accordingly. + # + # Inserts ordered content types into the environment as `ORDERED_CONTENT_TYPES` if an Accept header is present + # + # @param [Hash{String => String}] env + # @return [Array(Integer, Hash, #each)] Status, Headers and Body + # @see https://rubydoc.info/github/rack/rack/file/SPEC + def call(env) + if env['PATH_INFO'].match?(ENDPOINTS_FILTER) + if env.has_key?('HTTP_ACCEPT') + accepted_types = parse_accept_header(env['HTTP_ACCEPT']) + if !accepted_types.empty? + env["format"] = accepted_types.first + add_content_type_header(app.call(env), env["format"]) + else + not_acceptable + end + else + env["format"] = options[:default] + add_content_type_header(app.call(env), env["format"]) + end + else + app.call(env) + end + end + + protected + + # Parses an HTTP `Accept` header, returning an array of MIME content types ordered by precedence rules. + # + # @param [String, #to_s] header + # @return [Array] Array of content types sorted by precedence + # @see https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 + def parse_accept_header(header) + entries = header.to_s.split(',') + parsed_entries = entries.map { |entry| parse_accept_entry(entry) } + sorted_entries = parsed_entries.sort_by { |entry| entry.quality }.reverse + content_types = sorted_entries.map { |entry| entry.content_type } + content_types.flatten.compact + end + + + + # Parses an individual entry from the Accept header. + # + # @param [String] entry An entry from the Accept header + # @return [Entry] An object representing the parsed entry + def parse_accept_entry(entry) + # Represents an entry parsed from the Accept header + entry_struct = Struct.new(:content_type, :quality, :wildcard_count, :param_count) + content_type, *params = entry.split(';').map(&:strip) + quality = 1.0 # Default quality + params.reject! do |param| + if param.start_with?('q=') + quality = param[2..-1].to_f + true + end + end + wildcard_count = content_type.count('*') + entry_struct.new(content_type, quality, wildcard_count, params.size) + end + + + ## + # Returns a content type appropriate for the given `media_range`, + # returns `nil` if `media_range` contains a wildcard subtype + # that is not mapped. + # + # @param [String, #to_s] media_range + # @return [String, nil] + def find_content_type_for_media_range(media_range) + case media_range.to_s + when '*/*', 'text/*' + options[:default] + when 'application/n-triples' + 'application/n-triples' + when 'text/turtle' + 'text/turtle' + when 'application/json', 'application/ld+json', 'application/*' + 'application/ld+json' + when 'text/xml', 'text/rdf+xml', 'application/rdf+xml', 'application/xml' + 'application/rdf+xml' + else + nil + end + end + + ## + # Outputs an HTTP `406 Not Acceptable` response. + # + # @param [String, #to_s] message + # @return [Array(Integer, Hash, #each)] + def not_acceptable(message = nil) + code = 406 + http_status = [code, Rack::Utils::HTTP_STATUS_CODES[code]].join(' ') + message = http_status + (message.nil? ? "\n" : " (#{message})\n") + [code, { 'Content-Type' => "text/plain" }.merge(VARY), [message]] + end + + def add_content_type_header(response, type) + response[1] = response[1].merge(VARY).merge('Content-Type' => type) + response + end + + end +end diff --git a/models/simple_wrappers.rb b/models/simple_wrappers.rb index e4097aff..f6aeb027 100644 --- a/models/simple_wrappers.rb +++ b/models/simple_wrappers.rb @@ -29,3 +29,5 @@ ProvisionalRelation = LinkedData::Models::ProvisionalRelation SearchHelper = Sinatra::Helpers::SearchHelper + +Resource = LinkedData::Models::Resource \ No newline at end of file diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake new file mode 100644 index 00000000..52af504c --- /dev/null +++ b/rakelib/docker_based_test.rake @@ -0,0 +1,120 @@ +# Rake tasks for running unit tests with backend services running as docker containers + +desc 'Run unit tests with docker based backend' +namespace :test do + namespace :docker do + task :up do + system("docker compose up -d") || abort("Unable to start docker containers") + unless system("curl -sf http://localhost:8983/solr || exit 1") + printf("waiting for Solr container to initialize") + sec = 0 + until system("curl -sf http://localhost:8983/solr || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + abort(" Solr container hasn't initialized properly") + end + end + printf("\n") + end + end + task :down do + #system("docker compose --profile fs --profile ag stop") + #system("docker compose --profile fs --profile ag kill") + end + desc "run tests with docker AG backend" + task :ag do + ENV["GOO_BACKEND_NAME"]="allegrograph" + ENV["GOO_PORT"]="10035" + ENV["GOO_PATH_QUERY"]="/repositories/ontoportal_test" + ENV["GOO_PATH_DATA"]="/repositories/ontoportal_test/statements" + ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal_test/statements" + ENV["COMPOSE_PROFILES"]="ag" + Rake::Task["test:docker:up"].invoke + # AG takes some time to start and create databases/accounts + # TODO: replace system curl command with native ruby code + unless system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") + printf("waiting for AllegroGraph container to initialize") + sec = 0 + until system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") do + sleep(1) + printf(".") + sec += 1 + end + end + puts + system("docker compose ps") # TODO: remove after GH actions troubleshooting is complete + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + desc "run tests with docker 4store backend" + task :fs do + ENV["GOO_PORT"]="9000" + ENV["COMPOSE_PROFILES"]='fs' + Rake::Task["test:docker:up"].invoke + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + desc "run tests with docker Virtuoso backend" + task :vo do + ENV["GOO_BACKEND_NAME"]="virtuoso" + ENV["GOO_PORT"]="8890" + ENV["GOO_PATH_QUERY"]="/sparql" + ENV["GOO_PATH_DATA"]="/sparql" + ENV["GOO_PATH_UPDATE"]="/sparql" + ENV["COMPOSE_PROFILES"]="vo" + Rake::Task["test:docker:up"].invoke + # + unless system("curl -sf http://localhost:8890/sparql || exit 1") + printf("waiting for Virtuoso container to initialize") + sec = 0 + until system("curl -sf http://localhost:8890/sparql || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + system("docker compose logs virtuoso-ut") + abort(" Virtuoso container hasn't initialized properly") + end + end + end + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + + desc "run tests with docker GraphDb backend" + task :gb do + ENV["GOO_BACKEND_NAME"]="graphdb" + ENV["GOO_PORT"]="7200" + ENV["GOO_PATH_QUERY"]="/repositories/ontoportal" + ENV["GOO_PATH_DATA"]="/repositories/ontoportal/statements" + ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal/statements" + ENV["COMPOSE_PROFILES"]="gb" + Rake::Task["test:docker:up"].invoke + + #system("docker compose cp ./test/data/graphdb-repo-config.ttl graphdb:/opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl") + #system("docker compose cp ./test/data/graphdb-test-load.nt graphdb:/opt/graphdb/dist/configs/templates/graphdb-test-load.nt") + #system('docker compose exec graphdb sh -c "importrdf load -f -c /opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/graphdb-test-load.nt ;"') + unless system("curl -sf http://localhost:7200/repositories || exit 1") + printf("waiting for Graphdb container to initialize") + sec = 0 + until system("curl -sf http://localhost:7200/repositories || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + system("docker compose logs graphdb") + abort(" Graphdb container hasn't initialized properly") + end + end + end + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + end +end diff --git a/test/controllers/test_agents_controller.rb b/test/controllers/test_agents_controller.rb index de36bc36..658ef38b 100644 --- a/test/controllers/test_agents_controller.rb +++ b/test/controllers/test_agents_controller.rb @@ -168,24 +168,9 @@ def test_delete_agent end private + def _agent_data(type: 'organization') - schema_agencies = LinkedData::Models::AgentIdentifier::IDENTIFIER_SCHEMES.keys - users = LinkedData::Models::User.all - users = [LinkedData::Models::User.new(username: "tim", email: "tim@example.org", password: "password").save] if users.empty? - test_identifiers = 5.times.map { |i| { notation: rand.to_s[2..11], schemaAgency: schema_agencies.sample.to_s } } - user = users.sample.id.to_s - - i = rand.to_s[2..11] - return { - agentType: type, - name: "name #{i}", - homepage: "home page #{i}", - acronym: "acronym #{i}", - email: "email_#{i}@test.com", - identifiers: test_identifiers.sample(2).map { |x| x.merge({ creator: user }) }, - affiliations: [], - creator: user - } + agent_data(type: type) end def _find_agent(name) diff --git a/test/controllers/test_annotator_controller.rb b/test/controllers/test_annotator_controller.rb index 47f45f40..572c8750 100644 --- a/test/controllers/test_annotator_controller.rb +++ b/test/controllers/test_annotator_controller.rb @@ -16,7 +16,12 @@ def self.before_suite end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true, + process_options: { + process_rdf: true, + extract_metadata: false, + index_search: true + }) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) @@ -31,7 +36,7 @@ def test_annotate get "/annotator", params assert last_response.ok? annotations = MultiJson.load(last_response.body) - assert_equal(7, annotations.length) + assert_includes([7,6], annotations.length) text = < + + + + + + + + altération de l'ADN + + + + + + XML + + expected_result_2 = <<-XML + + + + + + + altération de l'ADN + + + + + + + + XML + + + clean_xml = -> (x) { x.strip.gsub('/>', '').gsub('', '').split(' ').reject(&:empty?)} + + + a = result.gsub('\\"', '"')[1..-2].split("\\n").map{|x| clean_xml.call(x)}.flatten + b_1 = expected_result_1.split("\n").map{|x| clean_xml.call(x)}.flatten + b_2 = expected_result_2.split("\n").map{|x| clean_xml.call(x)}.flatten + + assert_includes [b_1.sort, b_2.sort], a.sort + end + + def test_dereference_resource_controller_ntriples + header 'Accept', 'application/n-triples' + get "/ontologies/#{@@graph}/resolve/#{@@uri}" + assert last_response.ok? + + result = last_response.body + expected_result = <<-NTRIPLES + . + . + . + . + . + . + "alt\\u00E9rationdel'ADN"@fr . + . + NTRIPLES + a = result.gsub(' ', '').split("\n").reject(&:empty?) + b = expected_result.gsub(' ', '').split("\n").reject(&:empty?) + assert_equal b.sort, a.sort + end + + def test_dereference_resource_controller_turtle + header 'Accept', 'text/turtle' + get "/ontologies/#{@@graph}/resolve/#{@@uri}" + assert last_response.ok? + + result = last_response.body + expected_result = <<-TURTLE + @prefix rdf: . + @prefix ns0: . + @prefix owl: . + @prefix skos: . + + ns0:c_6496 + a owl:NamedIndividual, skos:Concept ; + skos:broader ns0:c_a9d99f3a ; + skos:inScheme ns0:mt_65, ns0:thesaurusINRAE ; + skos:prefLabel "altération de l'ADN"@fr ; + skos:topConceptOf ns0:mt_65 . + + ns0:mt_65 + skos:hasTopConcept ns0:c_6496 . + TURTLE + a = result.gsub(' ', '').split("\n").reject(&:empty?) + b = expected_result.gsub(' ', '').split("\n").reject(&:empty?) + + assert_equal b.sort, a.sort + end + + private + + def sort_nested_hash(hash) + sorted_hash = {} + + hash.each do |key, value| + if value.is_a?(Hash) + sorted_hash[key] = sort_nested_hash(value) + elsif value.is_a?(Array) + sorted_hash[key] = value.map { |item| item.is_a?(Hash) ? sort_nested_hash(item) : item }.sort_by { |item| item.to_s } + else + sorted_hash[key] = value + end + end + + sorted_hash.sort.to_h + end + +end \ No newline at end of file diff --git a/test/controllers/test_external_mappings_controller.rb b/test/controllers/test_external_mappings_controller.rb index 6cfabf32..cb1f255f 100644 --- a/test/controllers/test_external_mappings_controller.rb +++ b/test/controllers/test_external_mappings_controller.rb @@ -12,8 +12,10 @@ def self.before_suite ont.delete end end + # indexing term is needed LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: {process_rdf: true, extract_metadata: false, index_search: true}, acronym: "BRO-TEST-MAP", name: "BRO-TEST-MAP", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -22,6 +24,7 @@ def self.before_suite }) LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: {process_rdf: true, extract_metadata: false}, acronym: "CNO-TEST-MAP", name: "CNO-TEST-MAP", file_path: "./test/data/ontology_files/CNO_05.owl", @@ -30,6 +33,7 @@ def self.before_suite }) LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: {process_rdf: true, extract_metadata: false}, acronym: "FAKE-TEST-MAP", name: "FAKE-TEST-MAP", file_path: "./test/data/ontology_files/fake_for_mappings.owl", diff --git a/test/controllers/test_instances_controller.rb b/test/controllers/test_instances_controller.rb index 9560c0b0..e4b0460b 100644 --- a/test/controllers/test_instances_controller.rb +++ b/test/controllers/test_instances_controller.rb @@ -5,6 +5,7 @@ class TestInstancesController < TestCase def self.before_suite LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, generate_missing_labels: false}, acronym: 'XCT-TEST-INST', name: 'XCT-TEST-INST', file_path: './test/data/ontology_files/XCTontologyvtemp2.owl', @@ -13,9 +14,6 @@ def self.before_suite }) end - def self.after_suite - LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - end def test_first_default_page ont = Ontology.find('XCT-TEST-INST-0').include(:acronym).first @@ -52,6 +50,7 @@ def test_all_instance_pages assert last_response.ok? instance_count = instance_count + response['collection'].size end while response['nextPage'] + assert_equal 714, instance_count # Next page should have no results. diff --git a/test/controllers/test_mappings_controller.rb b/test/controllers/test_mappings_controller.rb index cff52225..736606de 100644 --- a/test/controllers/test_mappings_controller.rb +++ b/test/controllers/test_mappings_controller.rb @@ -13,8 +13,10 @@ def self.before_suite ont.delete end end + # indexing is needed LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, index_search: true}, acronym: "BRO-TEST-MAP", name: "BRO-TEST-MAP", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -23,6 +25,7 @@ def self.before_suite }) LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, index_search: true}, acronym: "CNO-TEST-MAP", name: "CNO-TEST-MAP", file_path: "./test/data/ontology_files/CNO_05.owl", @@ -31,6 +34,7 @@ def self.before_suite }) LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, index_search: true}, acronym: "FAKE-TEST-MAP", name: "FAKE-TEST-MAP", file_path: "./test/data/ontology_files/fake_for_mappings.owl", @@ -84,6 +88,7 @@ def test_mappings_file_load commun_created_mappings_test(created, mapping_term_a, mapping_term_b, relations) end + private def commun_created_mappings_test(created, mapping_term_a, mapping_term_b, relations) @@ -109,7 +114,7 @@ def commun_created_mappings_test(created, mapping_term_a, mapping_term_b, relati assert last_response.ok? mappings = MultiJson.load(last_response.body) mappings = mappings["collection"] - assert_equal 21, mappings.length + assert_includes [21,11], mappings.length rest_count = 0 mappings.each do |x| if x["process"] != nil @@ -152,7 +157,7 @@ def mappings_for_ontology assert mappings["prevPage"] == nil assert mappings["nextPage"] == nil - assert_equal 18, mappings["collection"].length + assert_includes [18,8], mappings["collection"].length mappings = mappings["collection"] mappings.each do |mapping| @@ -195,7 +200,7 @@ def mappings_between_ontologies assert mappings["prevPage"] == nil assert mappings["nextPage"] == nil - assert_equal 8, mappings["collection"].length + assert_includes [8,3], mappings["collection"].length mappings = mappings["collection"] mappings.each do |mapping| assert mapping["classes"].length, 2 @@ -419,4 +424,6 @@ def build_mappings_hash end [mappings, mapping_ont_a, mapping_ont_b, mapping_term_a, mapping_term_b, relations] end + + end diff --git a/test/controllers/test_metrics_controller.rb b/test/controllers/test_metrics_controller.rb index 1b8890a6..f5e3d5f3 100644 --- a/test/controllers/test_metrics_controller.rb +++ b/test/controllers/test_metrics_controller.rb @@ -7,22 +7,23 @@ def self.before_suite puts "this test is going to wipe out all submission and ontologies. probably this is not a test env." return end - OntologySubmission.all.each {|s| s.delete } - Ontology.all.each {|o| o.delete } - @@data = {"classes"=>486, - "averageChildCount"=>5, - "maxChildCount"=>65, - "classesWithOneChild"=>14, - "classesWithMoreThan25Children"=>2, - "classesWithNoDefinition"=>11, - "individuals"=>124, - "properties"=>63, - "maxDepth"=> 7 } - @@options = {ont_count: 2, - submission_count: 3, - submissions_to_process: [1, 2], - process_submission: true, - random_submission_count: false} + OntologySubmission.all.each { |s| s.delete } + Ontology.all.each { |o| o.delete } + @@data = { "classes" => [486, 481], # depending if owlapi imports SKOS + "averageChildCount" => 5, + "maxChildCount" => 65, + "classesWithOneChild" => [13, 14], + "classesWithMoreThan25Children" => 2, + "classesWithNoDefinition" => [11, 10], + "individuals" => 124, + "properties" => [63, 45], + "maxDepth" => 7 } + @@options = { ont_count: 2, + submission_count: 3, + submissions_to_process: [1, 2], + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, run_metrics: true, index_properties: true }, + random_submission_count: false } LinkedData::SampleData::Ontology.create_ontologies_and_submissions(@@options) end @@ -31,11 +32,15 @@ def test_all_metrics assert last_response.ok? metrics = MultiJson.load(last_response.body) assert metrics.length == 2 - #TODO: improve this test and test for two different ontologies - #though this is tested in LD + # TODO: improve this test and test for two different ontologies + # though this is tested in LD metrics.each do |m| - @@data.each do |k,v| - assert_equal(m[k], v) + @@data.each do |k, v| + if v.is_a?(Array) + assert_includes(v, m[k]) + else + assert_equal(v, m[k]) + end end assert m["@id"] == m["submission"].first + "/metrics" end @@ -46,10 +51,14 @@ def test_single_metrics get "/ontologies/#{ontology}/metrics" assert last_response.ok? metrics = MultiJson.load(last_response.body) - - @@data.each do |k,v| - assert_equal(metrics[k], v) + @@data.each do |k, v| + if v.is_a?(Array) + assert_includes(v, metrics[k]) + else + assert_equal(v, metrics[k]) + end end + end def test_metrics_with_submission_id @@ -57,9 +66,15 @@ def test_metrics_with_submission_id get "/ontologies/#{ontology}/submissions/1/metrics" assert last_response.ok? metrics = MultiJson.load(last_response.body) - @@data.each do |k,v| - assert_equal(metrics[k], v) + + @@data.each do |k, v| + if v.is_a?(Array) + assert_includes(v, metrics[k]) + else + assert_equal(v, metrics[k]) + end end + end def test_metrics_with_submission_id_as_param @@ -67,8 +82,12 @@ def test_metrics_with_submission_id_as_param get "/ontologies/#{ontology}/metrics?submissionId=1" assert last_response.ok? metrics = MultiJson.load(last_response.body) - @@data.each do |k,v| - assert_equal(metrics[k], v) + @@data.each do |k, v| + if v.is_a?(Array) + assert_includes(v, metrics[k]) + else + assert_equal(v, metrics[k]) + end end end @@ -78,18 +97,18 @@ def test_metrics_missing get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(0, ontologies.length, msg='Failure to detect 0 ontologies with missing metrics.') + assert_equal(0, ontologies.length, msg = 'Failure to detect 0 ontologies with missing metrics.') # create ontologies with latest submissions that have no metrics delete_ontologies_and_submissions - options = {ont_count: 2, - submission_count: 1, - process_submission: false, - random_submission_count: false} + options = { ont_count: 2, + submission_count: 1, + process_submission: false, + random_submission_count: false } create_ontologies_and_submissions(options) get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(2, ontologies.length, msg='Failure to detect 2 ontologies with missing metrics.') + assert_equal(2, ontologies.length, msg = 'Failure to detect 2 ontologies with missing metrics.') # recreate the before_suite data (this test might not be the last one to run in the suite) delete_ontologies_and_submissions create_ontologies_and_submissions(@@options) diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 34f8c4dc..ad062742 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -185,7 +185,9 @@ def test_download_ontology end def test_download_ontology_csv - num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) + num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, + process_submission: true, + process_options:{process_rdf: true, extract_metadata: true, index_search: true}) ont = onts.first acronym = created_ont_acronyms.first diff --git a/test/controllers/test_properties_controller.rb b/test/controllers/test_properties_controller.rb index 605ea385..38f8708f 100644 --- a/test/controllers/test_properties_controller.rb +++ b/test/controllers/test_properties_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", @@ -33,12 +35,12 @@ def test_properties get "/ontologies/#{@@acronyms.first}/properties" assert last_response.ok? results = MultiJson.load(last_response.body) - assert_equal 85, results.length + assert_includes [85, 56], results.length # depending if owlapi imports SKOS get "/ontologies/#{@@acronyms.last}/properties" assert last_response.ok? results = MultiJson.load(last_response.body) - assert_equal 35, results.length + assert_includes [35] , results.length # depending if owlapi imports SKOS end def test_single_property @@ -57,18 +59,19 @@ def test_property_roots get "/ontologies/#{@@acronyms.first}/properties/roots" assert last_response.ok? pr = MultiJson.load(last_response.body) - assert_equal 62, pr.length + assert_includes [62, 52], pr.length #depending if owlapi import SKOS # count object properties opr = pr.select { |p| p["@type"] == "http://www.w3.org/2002/07/owl#ObjectProperty" } - assert_equal 18, opr.length + assert_includes [18, 13], opr.length # count datatype properties dpr = pr.select { |p| p["@type"] == "http://www.w3.org/2002/07/owl#DatatypeProperty" } - assert_equal 32, dpr.length + assert_includes [32,31], dpr.length # count annotation properties apr = pr.select { |p| p["@type"] == "http://www.w3.org/2002/07/owl#AnnotationProperty" } - assert_equal 12, apr.length + assert_includes [12,8], apr.length # check for non-root properties + assert_empty pr.select { |p| ["http://www.w3.org/2004/02/skos/core#broaderTransitive", "http://www.w3.org/2004/02/skos/core#topConceptOf", "http://www.w3.org/2004/02/skos/core#relatedMatch", @@ -98,6 +101,10 @@ def test_property_roots end def test_property_tree + + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23topConceptOf" + return unless last_response.ok? # depending if owlapi import SKOS + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23topConceptOf/tree" assert last_response.ok? pr = MultiJson.load(last_response.body) @@ -129,6 +136,10 @@ def test_property_tree end def test_property_ancestors + + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23exactMatch" + return unless last_response.ok? + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23exactMatch/ancestors" assert last_response.ok? an = MultiJson.load(last_response.body) @@ -143,6 +154,9 @@ def test_property_ancestors end def test_property_descendants + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23note" + return unless last_response.ok? # depending if owlapi import SKOS + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23note/descendants" assert last_response.ok? dn = MultiJson.load(last_response.body) @@ -164,6 +178,9 @@ def test_property_descendants end def test_property_parents + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23changeNote" + return unless last_response.ok? # depending if owlapi import SKOS + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23changeNote/parents" assert last_response.ok? pr = MultiJson.load(last_response.body) @@ -189,6 +206,9 @@ def test_property_children ch = MultiJson.load(last_response.body) assert_empty ch + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23semanticRelation" + return unless last_response.ok? #depending if owlapi import SKOS + get "/ontologies/#{@@acronyms.first}/properties/http%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23semanticRelation/children" assert last_response.ok? ch = MultiJson.load(last_response.body) diff --git a/test/controllers/test_properties_search_controller.rb b/test/controllers/test_properties_search_controller.rb index f93a90a1..6c99fc40 100644 --- a/test/controllers/test_properties_search_controller.rb +++ b/test/controllers/test_properties_search_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesSearchController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", @@ -26,8 +28,8 @@ def self.before_suite def self.after_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - LinkedData::Models::Ontology.indexClear(:property) - LinkedData::Models::Ontology.indexCommit(nil, :property) + LinkedData::Models::OntologyProperty.indexClear + LinkedData::Models::OntologyProperty.indexCommit end def test_property_search @@ -55,7 +57,7 @@ def test_search_filters get '/property_search?q=has' assert last_response.ok? results = MultiJson.load(last_response.body) - assert_equal 17, results["collection"].length + assert_includes [17,4], results["collection"].length # depending if owlapi imports SKOS get '/property_search?q=has&ontologies=MCCLSEARCHTEST-0' assert last_response.ok? diff --git a/test/controllers/test_recommender_controller.rb b/test/controllers/test_recommender_controller.rb index 29caf28c..58d6d942 100644 --- a/test/controllers/test_recommender_controller.rb +++ b/test/controllers/test_recommender_controller.rb @@ -14,7 +14,7 @@ def self.before_suite @@redis.del(mappings) end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) diff --git a/test/controllers/test_recommender_v1_controller.rb b/test/controllers/test_recommender_v1_controller.rb index 7b14a63d..3ac4862d 100644 --- a/test/controllers/test_recommender_v1_controller.rb +++ b/test/controllers/test_recommender_v1_controller.rb @@ -1,10 +1,10 @@ require_relative '../test_case' -class TestRecommenderController < TestCase +class TestRecommenderV1Controller < TestCase def self.before_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) @@text = < "submissionAcronym:BROSEARCHTEST-0", :start => 0, :rows => 80}, :main) - #refute_equal 0, res["response"]["numFound"] - #refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + res = LinkedData::Models::Class.search("prefLabel_none:Activity", {:fq => "submissionAcronym:BROSEARCHTEST-0", :start => 0, :rows => 80}) + refute_equal 0, res["response"]["numFound"] + refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr" res = MultiJson.load(last_response.body) @@ -257,4 +264,5 @@ def test_multilingual_search end + end diff --git a/test/controllers/test_search_models_controller.rb b/test/controllers/test_search_models_controller.rb new file mode 100644 index 00000000..8f9a95fb --- /dev/null +++ b/test/controllers/test_search_models_controller.rb @@ -0,0 +1,470 @@ +require_relative '../test_case' + +class TestSearchModelsController < TestCase + + def self.after_suite + LinkedData::SampleData::Ontology.delete_ontologies_and_submissions + LinkedData::Models::Ontology.indexClear + LinkedData::Models::Agent.indexClear + LinkedData::Models::Class.indexClear + LinkedData::Models::OntologyProperty.indexClear + end + + def setup + self.class.after_suite + end + + def test_show_all_collection + get '/admin/search/collections' + assert last_response.ok? + res = MultiJson.load(last_response.body) + array = %w[agents_metadata ontology_data ontology_metadata prop_search_core1 term_search_core1] + assert_equal res["collections"].sort , array.sort + end + + def test_collection_schema + get '/admin/search/collections' + assert last_response.ok? + res = MultiJson.load(last_response.body) + collection = res["collections"].first + refute_nil collection + get "/admin/search/collections/#{collection}/schema" + assert last_response.ok? + res = MultiJson.load(last_response.body) + fields = res["fields"].map { |x| x["name"] } + assert_includes fields, 'id' + assert_includes fields, 'resource_id' + assert_includes fields, 'resource_model' + end + + def test_collection_search + + count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: false, + acronym: "BROSEARCHTEST", + name: "BRO Search Test", + file_path: "./test/data/ontology_files/BRO_v3.2.owl", + ont_count: 1, + submission_count: 1, + ontology_type: "VALUE_SET_COLLECTION" + }) + collection = 'ontology_metadata' + post "/admin/search/collections/#{collection}/search", {q: ""} + + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_equal 2, res['response']['numFound'] + end + + def test_search_security + count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, generate_missing_labels: false}, + acronym: "BROSEARCHTEST", + name: "BRO Search Test", + file_path: "./test/data/ontology_files/BRO_v3.2.owl", + ont_count: 1, + submission_count: 1, + ontology_type: "VALUE_SET_COLLECTION" + }) + + count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, generate_missing_labels: false}, + acronym: "MCCLSEARCHTEST", + name: "MCCL Search Test", + file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", + ont_count: 1, + submission_count: 1 + }) + + + subs = LinkedData::Models::OntologySubmission.all + subs.each do |s| + s.bring_remaining + s.index_all(Logger.new($stdout)) + end + + + allowed_user = User.new({ + username: "allowed", + email: "test1@example.org", + password: "12345" + }) + allowed_user.save + + blocked_user = User.new({ + username: "blocked", + email: "test2@example.org", + password: "12345" + }) + blocked_user.save + + bro = bro.first + bro.bring_remaining + bro.acl = [allowed_user] + bro.viewingRestriction = "private" + bro.save + + self.class.enable_security + get "/search/ontologies?query=#{bro.acronym}&apikey=#{blocked_user.apikey}" + response = MultiJson.load(last_response.body)["collection"] + assert_empty response.select{|x| x["ontology_acronym_text"].eql?(bro.acronym)} + + get "/search/ontologies/content?q=*Research_Lab_Management*&apikey=#{blocked_user.apikey}" + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_equal 0, res['totalCount'] + + get "/search/ontologies?query=#{bro.acronym}&apikey=#{allowed_user.apikey}" + response = MultiJson.load(last_response.body)["collection"] + refute_empty response.select{|x| x["ontology_acronym_text"].eql?(bro.acronym)} + + get "/search/ontologies/content?q=*Research_Lab_Management*&apikey=#{allowed_user.apikey}" + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_equal 1, res['totalCount'] + + self.class.reset_security(false) + end + + def test_ontology_metadata_search + count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: false, + acronym: "BROSEARCHTEST", + name: "BRO Search Test", + file_path: "./test/data/ontology_files/BRO_v3.2.owl", + ont_count: 1, + submission_count: 1, + ontology_type: "VALUE_SET_COLLECTION" + }) + + count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: false, + acronym: "MCCLSEARCHTEST", + name: "MCCL Search Test", + file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", + ont_count: 1, + submission_count: 1 + }) + + # Search ACRONYM + ## full word + get '/search/ontologies?query=BROSEARCHTEST-0' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + + ### start + get '/search/ontologies?query=BROSEARCHTEST' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + + ## part of the word + get '/search/ontologies?query=BRO' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + + + # Search name + ## full word + ### start + get '/search/ontologies?query=MCCL Search' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 'MCCLSEARCHTEST-0', response.first['ontology_acronym_text'] + ###in the middle + get '/search/ontologies?query=Search Test' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 2, response.size + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + assert_equal 'MCCLSEARCHTEST-0', response.last['ontology_acronym_text'] + ## part of the word + ### start + get '/search/ontologies?query=MCCL Sea' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 'MCCLSEARCHTEST-0', response.first['ontology_acronym_text'] + ### in the middle + get '/search/ontologies?query=Sea' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 2, response.size + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + assert_equal 'MCCLSEARCHTEST-0', response.last['ontology_acronym_text'] + + + ## full text + get '/search/ontologies?query=MCCL Search Test' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 'MCCLSEARCHTEST-0', response.first['ontology_acronym_text'] + + + # Search description + ## full word + ### start + get '/search/ontologies?query=Description' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 2, response.size + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + assert_equal 'MCCLSEARCHTEST-0', response.last['ontology_acronym_text'] + + ### in the middle + get '/search/ontologies?query=1' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 2, response.size + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + assert_equal 'MCCLSEARCHTEST-0', response.last['ontology_acronym_text'] + + ## part of the word + ### start + get '/search/ontologies?query=Desc' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 2, response.size + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + assert_equal 'MCCLSEARCHTEST-0', response.last['ontology_acronym_text'] + + ### full text + get '/search/ontologies?query=Description 1' + response = MultiJson.load(last_response.body)["collection"] + assert_equal 2, response.size + assert_equal 'BROSEARCHTEST-0', response.first['ontology_acronym_text'] + assert_equal 'MCCLSEARCHTEST-0', response.last['ontology_acronym_text'] + end + + def test_ontology_metadata_filters + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: 10, submission_count: 1) + + + group1 = LinkedData::Models::Group.find('group-1').first || LinkedData::Models::Group.new(acronym: 'group-1', name: "Test Group 1").save + group2 = LinkedData::Models::Group.find('group-2').first || LinkedData::Models::Group.new(acronym: 'group-2', name: "Test Group 2").save + category1 = LinkedData::Models::Category.find('category-1').first || LinkedData::Models::Category.new(acronym: 'category-1', name: "Test Category 1").save + category2 = LinkedData::Models::Category.find('category-2').first || LinkedData::Models::Category.new(acronym: 'category-2', name: "Test Category 2").save + + ontologies1 = ontologies[0..5].each do |o| + o.bring_remaining + o.group = [group1] + o.hasDomain = [category1] + o.save + end + + ontologies2 = ontologies[6..8].each do |o| + o.bring_remaining + o.group = [group2] + o.hasDomain = [category2] + o.save + end + + + # test filter by group and category + get "/search/ontologies?page=1&pagesize=100&groups=#{group1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size, MultiJson.load(last_response.body)["collection"].length + get "/search/ontologies?page=1&pagesize=100&groups=#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + + get "/search/ontologies?page=1&pagesize=100&groups=#{group1.acronym},#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies1.size + ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + get "/search/ontologies?page=1&pagesize=100&hasDomain=#{category1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size, MultiJson.load(last_response.body)["collection"].length + + get "/search/ontologies?page=1&pagesize=100&hasDomain=#{category2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + get "/search/ontologies?page=1&pagesize=100&hasDomain=#{category2.acronym},#{category1.acronym}" + assert last_response.ok? + assert_equal ontologies1.size + ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + get "/search/ontologies?page=1&pagesize=100&hasDomain=#{category2.acronym}&groups=#{group1.acronym}" + assert last_response.ok? + assert_equal 0, MultiJson.load(last_response.body)["collection"].length + get "/search/ontologies?page=1&pagesize=100&hasDomain=#{category2.acronym}&groups=#{group2.acronym}" + assert last_response.ok? + assert_equal ontologies2.size, MultiJson.load(last_response.body)["collection"].length + + + + ontologies3 = ontologies[9] + ontologies3.bring_remaining + ontologies3.group = [group1, group2] + ontologies3.hasDomain = [category1, category2] + ontologies3.name = "name search test" + ontologies3.save + + ontologies.first.name = "sort by test" + ontologies.first.save + sub = ontologies.first.latest_submission(status: :any).bring_remaining + sub.status = 'retired' + sub.description = "234" + sub.creationDate = DateTime.yesterday.to_datetime + sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first + sub.save + + #test search with sort + get "/search/ontologies?page=1&pagesize=100&q=tes&sort=ontology_name_sort asc" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + refute_empty submissions["collection"] + assert_equal ontologies.map{|x| x.bring(:name).name}.sort, submissions["collection"].map{|x| x["ontology_name_text"]} + + get "/search/ontologies?page=1&pagesize=100&q=tes&sort=creationDate_dt desc" + + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.map{|x| x.latest_submission(status: :any).bring(:creationDate).creationDate.to_s.split('T').first}.sort.reverse, + submissions["collection"].map{|x| x["creationDate_dt"].split('T').first} + + # test search with format + get "/search/ontologies?page=1&pagesize=100&q=tes&hasOntologyLanguage=SKOS" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + refute_empty submissions["collection"] + assert_equal 1, submissions["collection"].size + + + + get "/search/ontologies?page=1&pagesize=100&q=tes&hasOntologyLanguage=OWL" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.size-1 , submissions["collection"].size + + + # test ontology filter with submission filter attributes + get "/search/ontologies?page=1&pagesize=100&q=tes&groups=group-2&hasDomain=category-2&hasOntologyLanguage=OWL" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies2.size + 1 , submissions["collection"].size + + + + # test ontology filter with status + + get "/search/ontologies?page=1&pagesize=100&status=retired" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal 1 , submissions["collection"].size + + get "/search/ontologies?page=1&pagesize=100&status=alpha,beta,production" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + refute_empty submissions["collection"] + assert_equal ontologies.size - 1 , submissions["collection"].size + + get "/search/ontologies?page=1&pagesize=100&q=234" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal "http://data.bioontology.org/ontologies/TEST-ONT-0/submissions/1" , submissions["collection"].first["id"] + end + + def test_agents_search + agents_tmp = [ agent_data(type: 'organization'), agent_data(type: 'organization'), agent_data(type: 'person')] + agents_tmp.each do |a| + post "/agents", MultiJson.dump(a), "CONTENT_TYPE" => "application/json" + assert last_response.status == 201 + end + + agent_person = LinkedData::Models::Agent.where(agentType: 'person').all.first.bring_remaining + agent_org = LinkedData::Models::Agent.where(agentType: 'organization').all.first.bring_remaining + + + get "/search/agents?&q=name" + assert last_response.ok? + agents = MultiJson.load(last_response.body) + + + assert_equal 3, agents["totalCount"] + + + get "/search/agents?&q=name&agentType=organization" + assert last_response.ok? + agents = MultiJson.load(last_response.body) + assert_equal 2, agents["totalCount"] + + + + get "/search/agents?&q=name&agentType=person" + assert last_response.ok? + agents = MultiJson.load(last_response.body) + assert_equal 1, agents["totalCount"] + + + get "/search/agents?&q=#{agent_person.name}" + assert last_response.ok? + agents = MultiJson.load(last_response.body) + assert_equal agent_person.id.to_s, agents["collection"].first["id"] + + get "/search/agents?&q=#{agent_org.acronym}" + assert last_response.ok? + agents = MultiJson.load(last_response.body) + assert_equal agent_org.id.to_s, agents["collection"].first["id"] + + + get "/search/agents?&q=#{agent_org.identifiers.first.id.split('/').last}" + assert last_response.ok? + agents = MultiJson.load(last_response.body) + assert_equal agent_org.id.to_s, agents["collection"].first["id"] + end + + def test_search_data + count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, index_all_data: true, generate_missing_labels: false}, + acronym: "BROSEARCHTEST", + name: "BRO Search Test", + file_path: "./test/data/ontology_files/BRO_v3.2.owl", + ont_count: 1, + submission_count: 1, + ontology_type: "VALUE_SET_COLLECTION" + }) + + count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, index_all_data: true, generate_missing_labels: false}, + acronym: "MCCLSEARCHTEST", + name: "MCCL Search Test", + file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", + ont_count: 1, + submission_count: 1 + }) + + + subs = LinkedData::Models::OntologySubmission.all + count = [] + subs.each do |s| + count << Goo.sparql_query_client.query("SELECT (COUNT( DISTINCT ?id) as ?c) FROM <#{s.id}> WHERE {?id ?p ?v}") + .first[:c] + .to_i + end + + get "/search/ontologies/content?q=*" + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_equal count.sum, res['totalCount'] + + + get "/search/ontologies/content?q=*&ontologies=MCCLSEARCHTEST-0,BROSEARCHTEST-0" + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_equal count.sum, res['totalCount'] + + get "/search/ontologies/content?q=*&ontologies=BROSEARCHTEST-0" + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_includes count, res['totalCount'] + + get "/search/ontologies/content?q=*&ontologies=MCCLSEARCHTEST-0" + assert last_response.ok? + res = MultiJson.load(last_response.body) + assert_includes count, res['totalCount'] + + end +end diff --git a/test/controllers/test_slices_controller.rb b/test/controllers/test_slices_controller.rb index 92ce6b1d..601b15a7 100644 --- a/test/controllers/test_slices_controller.rb +++ b/test/controllers/test_slices_controller.rb @@ -3,28 +3,77 @@ class TestSlicesController < TestCase def self.before_suite - onts = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 0)[2] + ont_count, ont_acronyms, @@onts = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 0) @@slice_acronyms = ["tst-a", "tst-b"].sort - _create_slice(@@slice_acronyms[0], "Test Slice A", onts) - _create_slice(@@slice_acronyms[1], "Test Slice B", onts) + _create_slice(@@slice_acronyms[0], "Test Slice A", @@onts) + _create_slice(@@slice_acronyms[1], "Test Slice B", @@onts) + + @@user = User.new({ + username: "test-slice", + email: "test-slice@example.org", + password: "12345" + }).save + @@new_slice_data = { acronym: 'tst-c', name: "Test Slice C", ontologies: ont_acronyms} + @@old_security_setting = LinkedData.settings.enable_security + end + + def self.after_suite + LinkedData::Models::Slice.all.each(&:delete) + @@user.delete + reset_security(@@old_security_setting) + end + + def setup + self.class.reset_security(@@old_security_setting) + self.class.reset_to_not_admin(@@user) + LinkedData::Models::Slice.find(@@new_slice_data[:acronym]).first&.delete end def test_all_slices get "/slices" assert last_response.ok? slices = MultiJson.load(last_response.body) - assert_equal @@slice_acronyms, slices.map {|s| s["acronym"]}.sort + assert_equal @@slice_acronyms, slices.map { |s| s["acronym"] }.sort + end + + def test_create_slices + self.class.enable_security + + post "/slices?apikey=#{@@user.apikey}", MultiJson.dump(@@new_slice_data), "CONTENT_TYPE" => "application/json" + assert_equal 403, last_response.status + + self.class.make_admin(@@user) + + post "/slices?apikey=#{@@user.apikey}", MultiJson.dump(@@new_slice_data), "CONTENT_TYPE" => "application/json" + + assert 201, last_response.status + end + + def test_delete_slices + self.class.enable_security + LinkedData.settings.enable_security = @@old_security_setting + self.class._create_slice(@@new_slice_data[:acronym], @@new_slice_data[:name], @@onts) + + + delete "/slices/#{@@new_slice_data[:acronym]}?apikey=#{@@user.apikey}" + assert_equal 403, last_response.status + + self.class.make_admin(@@user) + + delete "/slices/#{@@new_slice_data[:acronym]}?apikey=#{@@user.apikey}" + assert 201, last_response.status end private def self._create_slice(acronym, name, ontologies) slice = LinkedData::Models::Slice.new({ - acronym: acronym, - name: "Test #{name}", - ontologies: ontologies - }) + acronym: acronym, + name: "Test #{name}", + ontologies: ontologies + }) slice.save end -end + +end \ No newline at end of file diff --git a/test/controllers/test_users_controller.rb b/test/controllers/test_users_controller.rb index 3710b503..a165a5d7 100644 --- a/test/controllers/test_users_controller.rb +++ b/test/controllers/test_users_controller.rb @@ -6,7 +6,7 @@ def self.before_suite @@usernames = %w(fred goerge henry ben mark matt charlie) # Create them again - @@usernames.each do |username| + @@users = @@usernames.map do |username| User.new(username: username, email: "#{username}@example.org", password: "pass_word").save end @@ -21,6 +21,17 @@ def self._delete_users end end + def test_admin_creation + existent_user = @@users.first #no admin + + refute _create_admin_user(apikey: existent_user.apikey), "A no admin user can't create an admin user or update it to an admin" + + existent_user = self.class.make_admin(existent_user) + assert _create_admin_user(apikey: existent_user.apikey), "Admin can create an admin user or update it to be an admin" + self.class.reset_to_not_admin(existent_user) + delete "/users/#{@@username}" + end + def test_all_users get '/users' assert last_response.ok? @@ -136,4 +147,32 @@ def test_oauth_authentication assert data[:email], user["email"] end end + + private + def _create_admin_user(apikey: nil) + user = {email: "#{@@username}@example.org", password: "pass_the_word", role: ['ADMINISTRATOR']} + LinkedData::Models::User.find(@@username).first&.delete + + put "/users/#{@@username}", MultiJson.dump(user), "CONTENT_TYPE" => "application/json", "Authorization" => "apikey token=#{apikey}" + assert last_response.status == 201 + created_user = MultiJson.load(last_response.body) + assert created_user["username"].eql?(@@username) + + get "/users/#{@@username}?apikey=#{apikey}" + assert last_response.ok? + user = MultiJson.load(last_response.body) + assert user["username"].eql?(@@username) + + return true if user["role"].eql?(['ADMINISTRATOR']) + + patch "/users/#{@@username}", MultiJson.dump(role: ['ADMINISTRATOR']), "CONTENT_TYPE" => "application/json", "Authorization" => "apikey token=#{apikey}" + assert last_response.status == 204 + + get "/users/#{@@username}?apikey=#{apikey}" + assert last_response.ok? + user = MultiJson.load(last_response.body) + assert user["username"].eql?(@@username) + + true if user["role"].eql?(['ADMINISTRATOR']) + end end diff --git a/test/data/graphdb-repo-config.ttl b/test/data/graphdb-repo-config.ttl new file mode 100644 index 00000000..9200da9a --- /dev/null +++ b/test/data/graphdb-repo-config.ttl @@ -0,0 +1,33 @@ +@prefix rdfs: . +@prefix rep: . +@prefix sail: . +@prefix xsd: . + +<#ontoportal> a rep:Repository; + rep:repositoryID "ontoportal"; + rep:repositoryImpl [ + rep:repositoryType "graphdb:SailRepository"; + [ + "http://example.org/owlim#"; + "false"; + ""; + "true"; + "false"; + "true"; + "true"; + "32"; + "10000000"; + ""; + "true"; + ""; + "0"; + "0"; + "false"; + "file-repository"; + "rdfsplus-optimized"; + "storage"; + "false"; + sail:sailType "owlim:Sail" + ] + ]; + rdfs:label "" . \ No newline at end of file diff --git a/test/data/graphdb-test-load.nt b/test/data/graphdb-test-load.nt new file mode 100644 index 00000000..e69de29b diff --git a/test/helpers/test_http_cache_helper.rb b/test/helpers/test_http_cache_helper.rb index 944198a6..5268066a 100644 --- a/test/helpers/test_http_cache_helper.rb +++ b/test/helpers/test_http_cache_helper.rb @@ -4,7 +4,6 @@ class TestHTTPCacheHelper < TestCaseHelpers def self.before_suite raise Exception, "Redis is unavailable, caching will not function" if LinkedData::HTTPCache.redis.ping.nil? - self.new("before_suite").delete_ontologies_and_submissions ontologies = self.new("before_suite")._ontologies @@ontology = ontologies.shift @@ontology_alt = ontologies.shift diff --git a/test/helpers/test_slices_helper.rb b/test/helpers/test_slices_helper.rb index 165a2a7e..bddd5c2d 100644 --- a/test/helpers/test_slices_helper.rb +++ b/test/helpers/test_slices_helper.rb @@ -70,6 +70,31 @@ def test_search_slices assert results.all? {|r| group_ids.include?(r["links"]["ontology"])} end + def test_mappings_slices + LinkedData::Mappings.create_mapping_counts(Logger.new(TestLogFile.new)) + + get "/mappings/statistics/ontologies/" + + expected_result_without_slice = ["PARSED-0", + "PARSED-1", + "http://data.bioontology.org/metadata/ExternalMappings", + "http://data.bioontology.org/metadata/InterportalMappings/agroportal", + "http://data.bioontology.org/metadata/InterportalMappings/ncbo", + "http://data.bioontology.org/metadata/InterportalMappings/sifr"] + + assert_equal expected_result_without_slice, MultiJson.load(last_response.body).keys.sort + + get "http://#{@@group_acronym}/mappings/statistics/ontologies/" + + expected_result_with_slice = ["PARSED-0", + "http://data.bioontology.org/metadata/ExternalMappings", + "http://data.bioontology.org/metadata/InterportalMappings/agroportal", + "http://data.bioontology.org/metadata/InterportalMappings/ncbo", + "http://data.bioontology.org/metadata/InterportalMappings/sifr"] + + assert_equal expected_result_with_slice, MultiJson.load(last_response.body).keys.sort + end + private def self._create_group diff --git a/test/middleware/test_rack_attack.rb b/test/middleware/test_rack_attack.rb index 0b10c9e1..53f5fe3b 100644 --- a/test/middleware/test_rack_attack.rb +++ b/test/middleware/test_rack_attack.rb @@ -32,8 +32,8 @@ def self.before_suite $stdout = File.open("/dev/null", "w") $stderr = File.open("/dev/null", "w") - # http://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic.2C_private_or_ephemeral_ports - @@port1 = Random.rand(55000..65535) + + @@port1 = self.new('').unused_port # Fork the process to create two servers. This isolates the Rack::Attack configuration, which makes other tests fail if included. @@pid1 = fork do @@ -45,7 +45,7 @@ def self.before_suite Signal.trap("HUP") { Process.exit! } end - @@port2 = Random.rand(55000..65535) # http://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic.2C_private_or_ephemeral_ports + @@port2 = self.new('').unused_port @@pid2 = fork do require_relative '../../config/rack_attack' Rack::Server.start( @@ -150,7 +150,7 @@ def request(user: nil, port: nil) # Sometimes a single request can get through without failing depending # on the order of the request as it coincides with the threaded requests. (LinkedData::OntologiesAPI.settings.req_per_second_per_ip * 2).times do - open("http://127.0.0.1:#{port}/ontologies", headers) + open("http://localhost:#{port}/ontologies", headers) end end diff --git a/test/test_case.rb b/test/test_case.rb index be162d5e..06bbc99f 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -74,12 +74,15 @@ def count_pattern(pattern) def backend_4s_delete if count_pattern("?s ?p ?o") < 400000 - LinkedData::Models::Ontology.where.include(:acronym).each do |o| - query = "submissionAcronym:#{o.acronym}" - LinkedData::Models::Ontology.unindexByQuery(query) + puts 'clear backend & index' + raise StandardError, 'Too many triples in KB, does not seem right to run tests' unless + count_pattern('?s ?p ?o') < 400000 + + graphs = Goo.sparql_query_client.query("SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o . } }") + graphs.each_solution do |sol| + Goo.sparql_data_client.delete_graph(sol[:g]) end - LinkedData::Models::Ontology.indexCommit() - Goo.sparql_update_client.update("DELETE {?s ?p ?o } WHERE { ?s ?p ?o }") + LinkedData::Models::SubmissionStatus.init_enum LinkedData::Models::OntologyType.init_enum LinkedData::Models::OntologyFormat.init_enum @@ -146,9 +149,33 @@ def app # @option options [TrueClass, FalseClass] :random_submission_count Use a random number of submissions between 1 and :submission_count # @option options [TrueClass, FalseClass] :process_submission Parse the test ontology file def create_ontologies_and_submissions(options = {}) + if options[:process_submission] && options[:process_options].nil? + options[:process_options] = { process_rdf: true, extract_metadata: false, generate_missing_labels: false } + end LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) end + + def agent_data(type: 'organization') + schema_agencies = LinkedData::Models::AgentIdentifier::IDENTIFIER_SCHEMES.keys + users = LinkedData::Models::User.all + users = [LinkedData::Models::User.new(username: "tim", email: "tim@example.org", password: "password").save] if users.empty? + test_identifiers = 5.times.map { |i| { notation: rand.to_s[2..11], schemaAgency: schema_agencies.sample.to_s } } + user = users.sample.id.to_s + + i = rand.to_s[2..11] + return { + agentType: type, + name: "name #{i}", + homepage: "home page #{i}", + acronym: "acronym #{i}", + email: "email_#{i}@test.com", + identifiers: test_identifiers.sample(2).map { |x| x.merge({ creator: user }) }, + affiliations: [], + creator: user + } + end + ## # Delete all ontologies and their submissions def delete_ontologies_and_submissions @@ -194,4 +221,45 @@ def get_errors(response) return errors.strip end + def self.enable_security + LinkedData.settings.enable_security = true + end + + def self.reset_security(old_security = @@old_security_setting) + LinkedData.settings.enable_security = old_security + end + + + def self.make_admin(user) + user.bring_remaining + user.role = [LinkedData::Models::Users::Role.find(LinkedData::Models::Users::Role::ADMIN).first] + user.save + end + + def self.reset_to_not_admin(user) + user.bring_remaining + user.role = [LinkedData::Models::Users::Role.find(LinkedData::Models::Users::Role::DEFAULT).first] + user.save + end + + def unused_port + max_retries = 5 + retries = 0 + server_port = Random.rand(55000..65535) + while port_in_use?(server_port) + retries += 1 + break if retries >= max_retries + server_port = Random.rand(55000..65535) + end + server_port + end + private + def port_in_use?(port) + server = TCPServer.new(port) + server.close + false + rescue Errno::EADDRINUSE + true + end + end