Merge branch 'master' into merge-to-master-2.3.4

Gemfile

@@ -27,7 +27,7 @@ gem 'rack-timeout'
 gem 'redis-rack-cache', '~> 2.0'
 
 # Data access (caching)
-gem 'redis'
+gem 'redis', '~> 4.8.1'
 gem 'redis-activesupport'
 gem 'redis-store', '1.9.1'
 
@@ -44,12 +44,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107
 gem 'redcarpet'
 
 # NCBO gems (can be from a local dev path or from rubygems/git)
-gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'development'
+gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'master'
 gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master'
 gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master'
 gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master'
 gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
-gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development'
+gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'master'
 
 group :development do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
@@ -74,4 +74,4 @@ group :test do
   gem 'simplecov', require: false
   gem 'simplecov-cobertura' # for codecov.io
   gem 'webmock'
-end
+end

Gemfile.lock

@@ -11,8 +11,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 5979402d5138850fb9bdb34edfa350e9af1b5d22
-  branch: development
+  revision: 74ea47defc7f6260b045a6c6997bbe6a59c7bf62
+  branch: master
   specs:
     goo (0.0.2)
       addressable (~> 2.8)
@@ -53,8 +53,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: a199eff007f5d7f18205d61194f3823445aa6460
-  branch: development
+  revision: 80a331d053ea04397a903452288c2186822c340c
+  branch: master
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -189,10 +189,10 @@ GEM
     httpclient (2.8.3)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json (2.6.3)
+    json (2.7.1)
     json-schema (2.8.1)
       addressable (>= 2.4)
-    json_pure (2.6.3)
+    json_pure (2.7.1)
     jwt (2.7.1)
     kgio (2.11.4)
     libxml-ruby (4.1.2)
@@ -387,7 +387,7 @@ DEPENDENCIES
   rack-timeout
   rake (~> 10.0)
   redcarpet
-  redis
+  redis (~> 4.8.1)
   redis-activesupport
   redis-rack-cache (~> 2.0)
   redis-store (= 1.9.1)

controllers/admin_controller.rb

@@ -68,7 +68,7 @@ class AdminController < ApplicationController
       latest = ont.latest_submission(status: :any)
       error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil?
       check_last_modified(latest)
-      latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
+      latest.bring(*submission_include_params)
       NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions)
       halt 204
     end
@@ -84,7 +84,7 @@ class AdminController < ApplicationController
         latest = ont.latest_submission(status: :any)
       end
       check_last_modified(latest) if latest
-      latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest
+      latest.bring(*submission_include_params) if latest
       reply(latest || {})
     end
 

controllers/classes_controller.rb

@@ -262,13 +262,7 @@ def includes_param_check
       end
     end
 
-    def concept_schemes
-      params["concept_schemes"]&.split(',') || []
-    end
 
-    def concept_collections
-      params["concept_collections"]&.split(',') || []
-    end
 
     def request_display(attrs)
 

controllers/ontologies_controller.rb

@@ -38,21 +38,12 @@ class OntologiesController < ApplicationController
       else
         latest = ont.latest_submission(status: :any)
       end
-      check_last_modified(latest) if latest
-      # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed)
+
       if latest
-        if includes_param.first == :all
-          # Bring what we need to display all attr of the submission
-          latest.bring_remaining
-          latest.bring({:contact=>[:name, :email],
-                      :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat,
-                                  :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType],
-                      :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]})
-        else
-          latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
-        end
+        check_last_modified(latest)
+        latest.bring(*submission_include_params)
       end
-      #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest
+
       reply(latest || {})
     end
 
@@ -62,7 +53,7 @@ class OntologiesController < ApplicationController
     patch '/:acronym/latest_submission' do
       ont = Ontology.find(params["acronym"]).first
       error 422, "You must provide an existing `acronym` to patch" if ont.nil?
-      
+
       submission = ont.latest_submission(status: :any)
 
       submission.bring(*OntologySubmission.attributes)

controllers/ontology_submissions_controller.rb

@@ -1,9 +1,15 @@
 class OntologySubmissionsController < ApplicationController
   get "/submissions" do
     check_last_modified_collection(LinkedData::Models::OntologySubmission)
-    #using appplication_helper method
-    options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")}
-    reply retrieve_latest_submissions(options).values
+    options = {
+               also_include_views: params["also_include_views"],
+               status: (params["include_status"] || "ANY")
+    }
+    subs = retrieve_latest_submissions(options)
+    subs = subs.values unless page?
+    # Force to show ontology reviews, notes and projects by default only for this request
+    LinkedData::Models::Ontology.serialize_default(*(LinkedData::Models::Ontology.hypermedia_settings[:serialize_default] + [:reviews, :notes, :projects]))
+    reply subs
   end
 
   ##
@@ -19,22 +25,18 @@ class OntologySubmissionsController < ApplicationController
     ##
     # Display all submissions of an ontology
     get do
-      ont = Ontology.find(params["acronym"]).include(:acronym).first
+      ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first
       error 422, "Ontology #{params["acronym"]} does not exist" unless ont
       check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym])
-      if includes_param.first == :all
-        # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed)
-        ont.bring(submissions: [:released, :creationDate, :status, :submissionId,
-                                {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat],
-                                 :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus])
-
-        ont.submissions.each do |sub|
-          sub.bring_remaining
-        end
-      else
-        ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param))
-      end
-      reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i }  # descending order of submissionId
+      check_access(ont)
+      options = {
+        also_include_views: true,
+        status: (params["include_status"] || "ANY"),
+        ontology: params["acronym"]
+      }
+      subs = retrieve_submissions(options)
+
+      reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i }  # descending order of submissionId
     end
 
     ##
@@ -53,7 +55,7 @@ class OntologySubmissionsController < ApplicationController
       ont.bring(:submissions)
       ont_submission = ont.submission(params["ontology_submission_id"])
       error 404, "`submissionId` not found" if ont_submission.nil?
-      ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
+      ont_submission.bring(*submission_include_params)
       reply ont_submission
     end
 

helpers/access_control_helper.rb

@@ -10,11 +10,7 @@ module AccessControlHelper
       def check_access(obj)
         return obj unless LinkedData.settings.enable_security
         if obj.is_a?(Enumerable)
-          if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on?
-            check_access(obj.first)
-          else
             filter_access(obj)
-          end
         else
           if obj.respond_to?(:read_restricted?) && obj.read_restricted?
             readable = obj.readable?(env["REMOTE_USER"])

helpers/application_helper.rb

@@ -31,6 +31,7 @@ def populate_from_params(obj, params)
           # Deal with empty strings for String and URI
           empty_string = value.is_a?(String) && value.empty?
           old_string_value_exists = obj.respond_to?(attribute) && (obj.send(attribute).is_a?(String) || obj.send(attribute).is_a?(RDF::URI))
+          old_string_value_exists = old_string_value_exists || (obj.respond_to?(attribute) && obj.send(attribute).is_a?(LinkedData::Models::Base))
           if old_string_value_exists && empty_string
             value = nil
           elsif empty_string
@@ -51,6 +52,10 @@ def populate_from_params(obj, params)
             value = is_arr ? value : [value]
             new_value = []
             value.each do |cls|
+              if uri_as_needed(cls["ontology"]).nil?
+                new_value << cls
+                next
+              end
               sub = LinkedData::Models::Ontology.find(uri_as_needed(cls["ontology"])).first.latest_submission
               new_value << LinkedData::Models::Class.find(cls["class"]).in(sub).first
             end
@@ -59,7 +64,7 @@ def populate_from_params(obj, params)
             # Replace the initial value with the object, handling Arrays as appropriate
             if value.is_a?(Array)
               value = value.map {|e| attr_cls.find(uri_as_needed(e)).include(attr_cls.attributes).first}
-            else
+            elsif !value.nil?
               value = attr_cls.find(uri_as_needed(value)).include(attr_cls.attributes).first
             end
           elsif attr_cls
@@ -355,40 +360,16 @@ def replace_url_prefix(id)
       end
 
       def retrieve_latest_submissions(options = {})
-        status = (options[:status] || "RDF").to_s.upcase
-        include_ready = status.eql?("READY") ? true : false
-        status = "RDF" if status.eql?("READY")
-        any = true if status.eql?("ANY")
-        include_views = options[:also_include_views] || false
-        includes = OntologySubmission.goo_attrs_to_load(includes_param)
-
-        includes << :submissionStatus unless includes.include?(:submissionStatus)
-        if any
-          submissions_query = OntologySubmission.where
-        else
-          submissions_query = OntologySubmission.where(submissionStatus: [ code: status])
-        end
+        submissions = retrieve_submissions(options)
 
-        submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views
-        submissions_query = submissions_query.filter(filter) if filter?
-        # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs
-        if includes_param.first == :all
-          includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl,
-                                       :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]
-        end
-        submissions = submissions_query.include(includes).to_a
-
-        # Figure out latest parsed submissions using all submissions
-        latest_submissions = {}
+        latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination
         submissions.each do |sub|
-          # To retrieve all metadata, but slow when a lot of ontologies
-          if includes_param.first == :all
-            sub.bring_remaining
+          unless page?
+            next if include_ready?(options) && !sub.ready?
+            next if sub.ontology.nil?
+            latest_submissions[sub.ontology.acronym] ||= sub
+            latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
           end
-          next if include_ready && !sub.ready?
-          next if sub.ontology.nil?
-          latest_submissions[sub.ontology.acronym] ||= sub
-          latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
         end
         latest_submissions
       end

helpers/classes_helper.rb

@@ -32,23 +32,19 @@ def get_class(submission, load_attrs=nil)
         load_children = load_attrs.delete :children
         load_has_children = load_attrs.delete :hasChildren
 
-        if !load_children
+        unless load_children
           load_children = load_attrs.select { |x| x.instance_of?(Hash) && x.include?(:children) }
-
-          if load_children.length == 0
-            load_children = nil
-          end
-          if !load_children.nil?
-            load_attrs = load_attrs.select { |x| !(x.instance_of?(Hash) && x.include?(:children)) }
-          end
+          load_children = nil if load_children.length == 0
+          load_attrs = load_attrs.select { |x| !(x.instance_of?(Hash) && x.include?(:children)) } unless load_children.nil?
         end
 
+
         cls_uri = notation_to_class_uri(submission)
 
         if cls_uri.nil?
           cls_uri = RDF::URI.new(params[:cls])
 
-          if !cls_uri.valid?
+          unless cls_uri.valid?
             error 400, "The input class id '#{params[:cls]}' is not a valid IRI"
           end
         end
@@ -62,23 +58,38 @@ def get_class(submission, load_attrs=nil)
           error 404,
                 "Resource '#{params[:cls]}' not found in ontology #{submission.ontology.acronym} submission #{submission.submissionId}"
         end
-        unless load_has_children.nil?
-          cls.load_has_children
-        end
 
-        if !load_children.nil?
+
+        extra_include = []
+
+        extra_include << :hasChildren if load_has_children
+        extra_include << :isInActiveScheme if load_attrs.include?(:inScheme)
+        extra_include << :isInActiveCollection if load_attrs.include?(:memberOf)
+
+        cls.load_computed_attributes(to_load: extra_include ,
+                                 options: {schemes: concept_schemes, collections: concept_collections})
+
+
+        unless load_children.nil?
           LinkedData::Models::Class.partially_load_children(
-            [cls],500,cls.submission)
+            [cls], 500, cls.submission)
           unless load_has_children.nil?
             cls.children.each do |c|
               c.load_has_children
             end
           end
         end
-        return cls
+        cls
       end
 
     end
+    def concept_schemes
+      params["concept_schemes"]&.split(',') || []
+    end
+
+    def concept_collections
+      params["concept_collections"]&.split(',') || []
+    end
   end
 end