Merge branch 'master' into merge-to-master-2.3.4

Gemfile

@@ -27,7 +27,7 @@ gem 'rack-timeout'
 gem 'redis-rack-cache', '~> 2.0'
 
 # Data access (caching)
-gem 'redis'
+gem 'redis', '~> 4.8.1'
 gem 'redis-activesupport'
 gem 'redis-store', '1.9.1'
 
@@ -44,12 +44,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107
 gem 'redcarpet'
 
 # NCBO gems (can be from a local dev path or from rubygems/git)
-gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'development'
+gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'master'
 gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master'
 gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master'
 gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master'
 gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
-gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development'
+gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'master'
 
 group :development do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
@@ -75,3 +75,4 @@ group :test do
   gem 'simplecov-cobertura' # for codecov.io
   gem 'webmock'
 end
+

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/ncbo_ontology_recommender.git
-  revision: 83e835de368bc9f19da800a477982e0ad770900d
+  revision: f440ae855a217807fead1d20629a0f187997b973
   branch: master
   specs:
     ncbo_ontology_recommender (0.0.1)
@@ -11,8 +11,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: ddb95e427950fde3ac715aec340394208c8166fe
-  branch: development
+  revision: bd7154217438c3b9160e0e9b495c7c718b55fbf8
+  branch: master
   specs:
     goo (0.0.2)
       addressable (~> 2.8)
@@ -53,8 +53,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 4c89c8346766d23e09b24c8e29750bf3a91e6b53
-  branch: development
+  revision: f44f7baa96eb3ee10dfab4a8aca154161ba7dd89
+  branch: master
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -105,14 +105,14 @@ GEM
       multi_json (~> 1.0)
     addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
-    airbrussh (1.4.2)
+    airbrussh (1.5.0)
       sshkit (>= 1.6.1, != 1.7.0)
     backports (3.24.1)
     bcrypt (3.1.19)
     bcrypt_pbkdf (1.1.0)
     bigdecimal (1.4.2)
     builder (3.2.4)
-    capistrano (3.17.3)
+    capistrano (3.18.0)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
@@ -159,7 +159,7 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
-    ffi (1.15.5)
+    ffi (1.16.3)
     get_process_mem (0.2.7)
       ffi (~> 1.0)
     google-apis-analytics_v3 (0.13.0)
@@ -173,10 +173,9 @@ GEM
       retriable (>= 2.0, < 4.a)
       rexml
       webrick
-    googleauth (1.7.0)
+    googleauth (1.8.1)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
-      memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
@@ -205,11 +204,11 @@ GEM
       net-imap
       net-pop
       net-smtp
-    memoist (0.16.2)
     method_source (1.0.0)
     mime-types (3.5.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0808)
+    mime-types-data (3.2023.1003)
     mini_mime (1.1.5)
     minitest (4.7.5)
     minitest-stub_any_instance (1.0.3)
@@ -218,7 +217,7 @@ GEM
     multi_json (1.15.0)
     multipart-post (2.3.0)
     net-http-persistent (2.9.4)
-    net-imap (0.3.7)
+    net-imap (0.4.1)
       date
       net-protocol
     net-pop (0.1.2)
@@ -227,11 +226,11 @@ GEM
       timeout
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.3.3)
+    net-smtp (0.4.0)
       net-protocol
     net-ssh (7.2.0)
     netrc (0.11.0)
-    newrelic_rpm (9.4.2)
+    newrelic_rpm (9.5.0)
     oj (2.18.5)
     omni_logger (0.1.4)
       logger
@@ -325,8 +324,8 @@ GEM
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
     systemu (2.6.5)
-    temple (0.10.2)
-    tilt (2.2.0)
+    temple (0.10.3)
+    tilt (2.3.0)
     timeout (0.4.0)
     trailblazer-option (0.1.2)
     tzinfo (2.0.6)
@@ -390,7 +389,7 @@ DEPENDENCIES
   rack-timeout
   rake (~> 10.0)
   redcarpet
-  redis
+  redis (~> 4.8.1)
   redis-activesupport
   redis-rack-cache (~> 2.0)
   redis-store (= 1.9.1)

controllers/admin_controller.rb

@@ -68,7 +68,7 @@ class AdminController < ApplicationController
       latest = ont.latest_submission(status: :any)
       error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil?
       check_last_modified(latest)
-      latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
+      latest.bring(*submission_include_params)
       NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions)
       halt 204
     end
@@ -84,7 +84,7 @@ class AdminController < ApplicationController
         latest = ont.latest_submission(status: :any)
       end
       check_last_modified(latest) if latest
-      latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest
+      latest.bring(*submission_include_params) if latest
       reply(latest || {})
     end
 

controllers/classes_controller.rb

@@ -262,13 +262,7 @@ def includes_param_check
       end
     end
 
-    def concept_schemes
-      params["concept_schemes"]&.split(',') || []
-    end
 
-    def concept_collections
-      params["concept_collections"]&.split(',') || []
-    end
 
     def request_display(attrs)
 

controllers/ontologies_controller.rb

@@ -38,21 +38,12 @@ class OntologiesController < ApplicationController
       else
         latest = ont.latest_submission(status: :any)
       end
-      check_last_modified(latest) if latest
-      # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed)
+
       if latest
-        if includes_param.first == :all
-          # Bring what we need to display all attr of the submission
-          latest.bring_remaining
-          latest.bring({:contact=>[:name, :email],
-                      :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat,
-                                  :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType],
-                      :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]})
-        else
-          latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
-        end
+        check_last_modified(latest)
+        latest.bring(*submission_include_params)
       end
-      #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest
+
       reply(latest || {})
     end
 
@@ -62,7 +53,7 @@ class OntologiesController < ApplicationController
     patch '/:acronym/latest_submission' do
       ont = Ontology.find(params["acronym"]).first
       error 422, "You must provide an existing `acronym` to patch" if ont.nil?
-      
+
       submission = ont.latest_submission(status: :any)
 
       submission.bring(*OntologySubmission.attributes)

controllers/ontology_submissions_controller.rb

@@ -1,9 +1,15 @@
 class OntologySubmissionsController < ApplicationController
   get "/submissions" do
     check_last_modified_collection(LinkedData::Models::OntologySubmission)
-    #using appplication_helper method
-    options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")}
-    reply retrieve_latest_submissions(options).values
+    options = {
+               also_include_views: params["also_include_views"],
+               status: (params["include_status"] || "ANY")
+    }
+    subs = retrieve_latest_submissions(options)
+    subs = subs.values unless page?
+    # Force to show ontology reviews, notes and projects by default only for this request
+    LinkedData::Models::Ontology.serialize_default(*(LinkedData::Models::Ontology.hypermedia_settings[:serialize_default] + [:reviews, :notes, :projects]))
+    reply subs
   end
 
   ##
@@ -19,22 +25,18 @@ class OntologySubmissionsController < ApplicationController
     ##
     # Display all submissions of an ontology
     get do
-      ont = Ontology.find(params["acronym"]).include(:acronym).first
+      ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first
       error 422, "Ontology #{params["acronym"]} does not exist" unless ont
       check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym])
-      if includes_param.first == :all
-        # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed)
-        ont.bring(submissions: [:released, :creationDate, :status, :submissionId,
-                                {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat],
-                                 :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus])
-
-        ont.submissions.each do |sub|
-          sub.bring_remaining
-        end
-      else
-        ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param))
-      end
-      reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i }  # descending order of submissionId
+      check_access(ont)
+      options = {
+        also_include_views: true,
+        status: (params["include_status"] || "ANY"),
+        ontology: params["acronym"]
+      }
+      subs = retrieve_submissions(options)
+
+      reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i }  # descending order of submissionId
     end
 
     ##
@@ -53,7 +55,7 @@ class OntologySubmissionsController < ApplicationController
       ont.bring(:submissions)
       ont_submission = ont.submission(params["ontology_submission_id"])
       error 404, "`submissionId` not found" if ont_submission.nil?
-      ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
+      ont_submission.bring(*submission_include_params)
       reply ont_submission
     end
 

helpers/access_control_helper.rb

@@ -10,11 +10,7 @@ module AccessControlHelper
       def check_access(obj)
         return obj unless LinkedData.settings.enable_security
         if obj.is_a?(Enumerable)
-          if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on?
-            check_access(obj.first)
-          else
             filter_access(obj)
-          end
         else
           if obj.respond_to?(:read_restricted?) && obj.read_restricted?
             readable = obj.readable?(env["REMOTE_USER"])

helpers/application_helper.rb

@@ -31,6 +31,7 @@ def populate_from_params(obj, params)
           # Deal with empty strings for String and URI
           empty_string = value.is_a?(String) && value.empty?
           old_string_value_exists = obj.respond_to?(attribute) && (obj.send(attribute).is_a?(String) || obj.send(attribute).is_a?(RDF::URI))
+          old_string_value_exists = old_string_value_exists || (obj.respond_to?(attribute) && obj.send(attribute).is_a?(LinkedData::Models::Base))
           if old_string_value_exists && empty_string
             value = nil
           elsif empty_string
@@ -51,6 +52,10 @@ def populate_from_params(obj, params)
             value = is_arr ? value : [value]
             new_value = []
             value.each do |cls|
+              if uri_as_needed(cls["ontology"]).nil?
+                new_value << cls
+                next
+              end
               sub = LinkedData::Models::Ontology.find(uri_as_needed(cls["ontology"])).first.latest_submission
               new_value << LinkedData::Models::Class.find(cls["class"]).in(sub).first
             end
@@ -59,7 +64,7 @@ def populate_from_params(obj, params)
             # Replace the initial value with the object, handling Arrays as appropriate
             if value.is_a?(Array)
               value = value.map {|e| attr_cls.find(uri_as_needed(e)).include(attr_cls.attributes).first}
-            else
+            elsif !value.nil?
               value = attr_cls.find(uri_as_needed(value)).include(attr_cls.attributes).first
             end
           elsif attr_cls
@@ -355,40 +360,16 @@ def replace_url_prefix(id)
       end
 
       def retrieve_latest_submissions(options = {})
-        status = (options[:status] || "RDF").to_s.upcase
-        include_ready = status.eql?("READY") ? true : false
-        status = "RDF" if status.eql?("READY")
-        any = true if status.eql?("ANY")
-        include_views = options[:also_include_views] || false
-        includes = OntologySubmission.goo_attrs_to_load(includes_param)
-
-        includes << :submissionStatus unless includes.include?(:submissionStatus)
-        if any
-          submissions_query = OntologySubmission.where
-        else
-          submissions_query = OntologySubmission.where(submissionStatus: [ code: status])
-        end
+        submissions = retrieve_submissions(options)
 
-        submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views
-        submissions_query = submissions_query.filter(filter) if filter?
-        # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs
-        if includes_param.first == :all
-          includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl,
-                                       :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]
-        end
-        submissions = submissions_query.include(includes).to_a
-
-        # Figure out latest parsed submissions using all submissions
-        latest_submissions = {}
+        latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination
         submissions.each do |sub|
-          # To retrieve all metadata, but slow when a lot of ontologies
-          if includes_param.first == :all
-            sub.bring_remaining
+          unless page?
+            next if include_ready?(options) && !sub.ready?
+            next if sub.ontology.nil?
+            latest_submissions[sub.ontology.acronym] ||= sub
+            latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
           end
-          next if include_ready && !sub.ready?
-          next if sub.ontology.nil?
-          latest_submissions[sub.ontology.acronym] ||= sub
-          latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
         end
         latest_submissions
       end