Merge branch 'fix/search-multilingual-filter' into stage

Dockerfile

@@ -23,6 +23,7 @@ RUN bundle install
 
 COPY . /srv/ontoportal/ontologies_api
 RUN cp /srv/ontoportal/ontologies_api/config/environments/config.rb.sample /srv/ontoportal/ontologies_api/config/environments/development.rb
+RUN cp /srv/ontoportal/ontologies_api/config/environments/config.rb.sample /srv/ontoportal/ontologies_api/config/environments/production.rb
 
 EXPOSE 9393
 CMD ["bundle", "exec", "rackup", "-p", "9393", "--host", "0.0.0.0"]

Gemfile

@@ -6,7 +6,7 @@ gem 'bigdecimal', '1.4.2'
 gem 'faraday', '~> 1.9'
 gem 'json-schema', '~> 2.0'
 gem 'multi_json', '~> 1.0'
-gem 'oj', '~> 2.0'
+gem 'oj'
 gem 'parseconfig'
 gem 'rack'
 gem 'rake', '~> 10.0'
@@ -47,7 +47,7 @@ gem 'redcarpet'
 
 # NCBO gems (can be from a local dev path or from rubygems/git)
 gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development'
-gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'development'
+gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master'
 gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master'
 gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development'
 gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'
@@ -77,4 +77,4 @@ group :test do
   gem 'simplecov', require: false
   gem 'simplecov-cobertura' # for codecov.io
   gem 'webmock', '~> 3.19.1'
-end
+end

Gemfile.lock

@@ -40,8 +40,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ncbo_cron.git
-  revision: 310ae5fef858d13917ff9fa6e59859998220c9b5
-  branch: development
+  revision: 6bb53a13f514a60513afe25e37c5c69475140452
+  branch: master
   specs:
     ncbo_cron (0.0.1)
       dante
@@ -114,6 +114,7 @@ GEM
     base64 (0.2.0)
     bcrypt (3.1.20)
     bcrypt_pbkdf (1.1.1)
+    bcrypt_pbkdf (1.1.1-x86_64-darwin)
     bigdecimal (1.4.2)
     builder (3.3.0)
     capistrano (3.19.0)
@@ -197,6 +198,8 @@ GEM
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
     google-cloud-errors (1.4.0)
+    google-protobuf (3.25.3)
+    google-protobuf (3.25.3-x86_64-darwin)
     google-protobuf (3.25.3-x86_64-linux)
     googleapis-common-protos (1.6.0)
       google-protobuf (>= 3.18, < 5.a)
@@ -211,6 +214,12 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
+    grpc (1.64.0)
+      google-protobuf (~> 3.25)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.64.0-x86_64-darwin)
+      google-protobuf (~> 3.25)
+      googleapis-common-protos-types (~> 1.0)
     grpc (1.64.0-x86_64-linux)
       google-protobuf (~> 3.25)
       googleapis-common-protos-types (~> 1.0)
@@ -273,7 +282,7 @@ GEM
     net-ssh (7.2.3)
     netrc (0.11.0)
     newrelic_rpm (9.11.0)
-    oj (2.18.5)
+    oj (3.16.1)
     omni_logger (0.1.4)
       logger
     os (1.1.4)
@@ -405,6 +414,8 @@ GEM
       hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
+  ruby
+  x86_64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -430,7 +441,7 @@ DEPENDENCIES
   ncbo_cron!
   ncbo_ontology_recommender!
   newrelic_rpm
-  oj (~> 2.0)
+  oj
   ontologies_linked_data!
   parallel
   parseconfig
@@ -462,4 +473,4 @@ DEPENDENCIES
   webmock (~> 3.19.1)
 
 BUNDLED WITH
-   2.3.15
+   2.4.22

config/environments/config.rb.sample

@@ -19,8 +19,8 @@ REDIS_PORT            = ENV.include?("REDIS_PORT")            ? ENV["REDIS_PORT"
 REPORT_PATH           = ENV.include?("REPORT_PATH")           ? ENV["REPORT_PATH"]           : "./test/ontologies_report.json"
 REPOSITORY_FOLDER     = ENV.include?("REPOSITORY_FOLDER")     ? ENV["REPOSITORY_FOLDER"]     : "./test/data/ontology_files/repo"
 REST_URL_PREFIX       = ENV.include?("REST_URL_PREFIX")       ? ENV["REST_URL_PREFIX"]       : ENV["API_URL"] || "http://localhost:9393"
-SOLR_PROP_SEARCH_URL  = ENV.include?("SOLR_PROP_SEARCH_URL")  ? ENV["SOLR_PROP_SEARCH_URL"]  : "http://localhost:8983/solr/prop_search_core1"
-SOLR_TERM_SEARCH_URL  = ENV.include?("SOLR_TERM_SEARCH_URL")  ? ENV["SOLR_TERM_SEARCH_URL"]  : "http://localhost:8983/solr/term_search_core1"
+SOLR_PROP_SEARCH_URL  = ENV.include?("SOLR_PROP_SEARCH_URL")  ? ENV["SOLR_PROP_SEARCH_URL"]  : "http://localhost:8983/solr"
+SOLR_TERM_SEARCH_URL  = ENV.include?("SOLR_TERM_SEARCH_URL")  ? ENV["SOLR_TERM_SEARCH_URL"]  : "http://localhost:8983/solr"
 
 begin
   # For prefLabel extract main_lang first, or anything if no main found.
@@ -49,6 +49,7 @@ LinkedData.config do |config|
   config.replace_url_prefix            = true
   config.rest_url_prefix               = REST_URL_PREFIX.to_s
   config.sparql_endpoint_url = "http://sparql.bioontology.org"
+  config.repository_folder            = REPOSITORY_FOLDER.to_s
 #  config.enable_notifications          = false
 
     config.interportal_hash = {

config/unicorn.rb

@@ -1,23 +1,45 @@
 application  = 'ontologies_api'
 app_path = "/srv/ontoportal/#{application}"
-working_directory "#{app_path}/current/"
+current_version_path = "#{app_path}/current"
+pid_file_path = 'tmp/pids/unicorn.pid'
 
+if Dir.exists?(current_version_path)
+  app_socket_path = app_path + '/shared/tmp/sockets/unicorn.sock'
+  app_gemfile_path = "#{current_version_path}/Gemfile"
+  user = 'ontoportal'
+else
+  current_version_path = app_path
+  app_gemfile_path = "#{app_path}/Gemfile"
+  app_socket_path = app_path + '/tmp/sockets/unicorn.sock'
+  user = 'root'
+end
+
+working_directory current_version_path
 worker_processes 8
 timeout 300
 preload_app true
-user 'ontoportal', 'ontoportal'
+user user, user
 
 stderr_path 'log/unicorn.stderr.log'
 stdout_path 'log/unicorn.stdout.log'
-pid 'tmp/pids/unicorn.pid'
 
+
+require 'fileutils'
+[pid_file_path, app_socket_path].each do |file_path|
+  directory_path = File.dirname(file_path)
+  FileUtils.mkdir_p(directory_path) unless Dir.exist?(File.dirname(file_path))
+end
+
+
+
+pid  pid_file_path
 # Listen on both fast-failing unix data socket (for nginx) & a backloggable TCP connection
-listen app_path + '/shared/tmp/sockets/unicorn.sock', :backlog => 1024
+listen app_socket_path, :backlog => 1024
 #listen 8087, :backlog => 256
 
 # Make sure unicorn is using current gems after rolling restarts
 before_exec do |server|
-  ENV['BUNDLE_GEMFILE'] = "#{app_path}/current/Gemfile"
+  ENV['BUNDLE_GEMFILE'] =  app_gemfile_path
 end
 
 before_fork do |server, worker|

controllers/home_controller.rb

@@ -121,7 +121,7 @@ def metadata_all
               unique: cls.unique?(attribute) || " ",
               required: cls.required?(attribute) || " ",
               list: cls.list?(attribute) || " ",
-              cardinality: cls.cardinality(attribute) || " "
+              cardinality: (cls.cardinality(attribute) rescue nil) || " "
             }
           else
             attributes_info[attribute] = {

controllers/mappings_controller.rb

@@ -191,6 +191,8 @@ class MappingsController < ApplicationController
                                       .each do |m|
         persistent_counts[m.ontologies.first] = m.count
       end
+      ont_acronyms = restricted_ontologies_to_acronyms(params)
+      persistent_counts = persistent_counts.select { |key, _| ont_acronyms.include?(key) || key.start_with?("http://") }
       reply persistent_counts
     end
 

controllers/search_controller.rb

@@ -230,6 +230,9 @@ def process_search(params = nil)
         doc[:submission] = submission
         doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0
         doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties)
+
+        doc = filter_attrs_by_language(doc)
+
         instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc)
         docs.push(instance)
       end

docker-compose.yml

@@ -1,31 +1,27 @@
 x-app: &app
-  image: agroportal/ontologies_api:development
+  image: agroportal/ontologies_api:master
   environment: &env
     # default bundle config resolves to /usr/local/bundle/config inside of the container
     # we are setting it to local app directory if we need to use 'bundle config local'
-    BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle
     BUNDLE_PATH: /srv/ontoportal/bundle
     COVERAGE: 'true' # enable simplecov code coverage
     REDIS_HOST: redis-ut
     REDIS_PORT: 6379
     SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr
     SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr
+    GOO_BACKEND_NAME: 4store
+    GOO_PORT: 9000
+    GOO_HOST: 4store-ut
+    MGREP_HOST: mgrep-ut
+    MGREP_PORT: 55555
+    REPOSITORY_FOLDER: /srv/ontoportal/data/repository
+    REPORT_PATH: /srv/ontoportal/data/reports/ontologies_report.json
+    MGREP_DICTIONARY_FILE: /srv/ontoportal/data/mgrep
   stdin_open: true
   tty: true
   command: /bin/bash
-  volumes:
-    # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development
-    - bundle:/srv/ontoportal/bundle
-    - .:/srv/ontoportal/ontologies_api
-    # mount directory containing development version of the gems if you need to use 'bundle config local'
-    #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal
-  depends_on: &depends_on
-    solr-prop-ut:
-      condition: service_healthy
-    solr-term-ut:
-      condition: service_healthy
-    redis-ut:
-      condition: service_healthy
+
+
 
 services:
   api:
@@ -34,21 +30,55 @@ services:
       .env
     environment:
       <<: *env
-      GOO_BACKEND_NAME: 4store
-      GOO_PORT: 9000
-      GOO_HOST: 4store-ut
-      GOO_PATH_QUERY: /sparql/
-      GOO_PATH_DATA: /data/
-      GOO_PATH_UPDATE: /update/
+      BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle
     profiles:
       - 4store
     depends_on:
-      - solr-ut
-      - redis-ut
-      - mgrep-ut
-      - 4store-ut
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      mgrep-ut:
+        condition: service_started
+      4store-ut:
+        condition: service_started
+      ncbo_cron:
+        condition: service_started
     ports:
       - "9393:9393"
+    volumes:
+      # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development
+      - app_api:/srv/ontoportal/ontologies_api
+      - repository:/srv/ontoportal/data/repository
+
+  ncbo_cron:
+    <<: *app
+    image: agroportal/ncbo_cron:master
+    env_file:
+      .env
+    environment:
+      <<: *env
+      BUNDLE_APP_CONFIG: /srv/ontoportal/ncbo_cron/.bundle
+    command: "bundle exec bin/ncbo_cron"
+    profiles:
+      - 4store
+    volumes:
+      - app_cron:/srv/ontoportal/ncbo_cron
+      - repository:/srv/ontoportal/data/repository
+      - history:/usr/local/hist
+      - reports:/srv/ontoportal/data/reports
+      - mgrep:/srv/ontoportal/data/mgrep
+      - logs:/srv/ontoportal/ncbo_cron/logs
+    depends_on:
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      mgrep-ut:
+        condition: service_started
+      4store-ut:
+        condition: service_started
+
 
   mgrep-ut:
     image: ontoportal/mgrep-ncbo:0.1
@@ -84,8 +114,14 @@ services:
     ports:
       - 8983:8983
     command: bin/solr start -cloud -f
-   # volumes:
-      #- solr_data:/var/solr/data
+    volumes:
+      - solr_data:/var/solr/data
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8983/solr/admin/info/system?wt=json" ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
   agraph-ut:
     image: franzinc/agraph:v8.1.0
     platform: linux/amd64
@@ -152,7 +188,13 @@ services:
       - gb
 
 volumes:
-  bundle:
+  app_api:
+  app_cron:
   agdata:
   4store:
-  #solr_data:
+  repository:
+  solr_data:
+  reports:
+  mgrep:
+  logs:
+  history:

helpers/search_helper.rb

@@ -221,6 +221,37 @@ def add_matched_fields(solr_response, default_match)
         solr_response["match_types"] = all_matches
       end
 
+      def portal_language
+        Goo.main_languages.first
+      end
+
+      def request_language
+        params['lang'] || params['languages'] || portal_language
+      end
+
+
+      def filter_attrs_by_language(doc)
+        lang_values = {}
+        doc.each do |k, v|
+          attr, lang = k.to_s.split('_')
+
+          next unless lang
+
+          if lang.eql?('none') || request_language.eql?(lang)
+            lang_values[attr.to_sym] ||= []
+            lang_values[attr.to_sym] = lang.eql?('none') ? lang_values[attr.to_sym] + v : v + lang_values[attr.to_sym]
+          end
+        end
+
+        lang_values.each do |k, v|
+          doc[k] = v unless v.empty?
+        end
+
+        doc[:prefLabel] = doc["prefLabel_#{request_language}".to_sym]&.first || doc[:prefLabel]&.first
+        doc
+      end
+
+
       # see https://github.com/rsolr/rsolr/issues/101
       # and https://github.com/projecthydra/active_fedora/commit/75b4afb248ee61d9edb56911b2ef51f30f1ce17f
       #
@@ -333,7 +364,7 @@ def populate_classes_from_search(classes, ontology_acronyms=nil)
         params["fq"] << " AND #{get_quoted_field_query_param(class_ids, "OR", "resource_id")}"
         params["rows"] = 99999
         # Replace fake query with wildcard
-        resp = LinkedData::Models::Class.search("*:*", params)
+        resp = LinkedData::Models::Class.submit_search_query("*:*", params)
 
         classes_hash = {}
         resp["response"]["docs"].each do |doc|