Merge pull request #22 from ontoportal-lirmm/development

Merge to master: Release 2.5.0 - Refactor cache, federated calls, and Rails performance logger

Gemfile.lock

@@ -2,33 +2,35 @@ PATH
   remote: .
   specs:
     ontologies_api_client (2.2.0)
-      activesupport
-      excon
+      activesupport (~> 7.0.4)
+      excon (= 0.112.0)
       faraday
       faraday-excon (~> 2.0.0)
       faraday-multipart
       lz4-ruby
       multi_json
       oj
+      parallel
+      request_store
       spawnling (= 2.1.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.4)
+    activesupport (7.0.8.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
-    bigdecimal (3.1.7)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    bigdecimal (3.1.8)
     coderay (1.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.3.4)
     crack (1.0.0)
       bigdecimal
       rexml
-    excon (0.95.0)
+    excon (0.112.0)
     faraday (2.0.1)
       faraday-net_http (~> 2.0)
       ruby2_keywords (>= 0.0.4)
@@ -38,34 +40,42 @@ GEM
     faraday-multipart (1.0.4)
       multipart-post (~> 2)
     faraday-net_http (2.1.0)
-    hashdiff (1.1.0)
-    i18n (1.12.0)
+    hashdiff (1.1.1)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
     lz4-ruby (0.3.3)
-    method_source (1.0.0)
-    minitest (5.16.3)
+    method_source (1.1.0)
+    minitest (5.25.1)
     multi_json (1.15.0)
-    multipart-post (2.2.3)
-    oj (3.13.23)
-    power_assert (2.0.2)
-    pry (0.14.1)
+    multipart-post (2.4.1)
+    oj (3.16.6)
+      bigdecimal (>= 3.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
+    parallel (1.26.3)
+    power_assert (2.0.4)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.5)
-    rake (13.0.6)
-    rexml (3.2.6)
+    public_suffix (5.1.1)
+    rack (3.1.8)
+    rake (13.2.1)
+    request_store (1.7.0)
+      rack (>= 1.4)
+    rexml (3.3.9)
     ruby2_keywords (0.0.5)
     spawnling (2.1.5)
-    test-unit (3.5.7)
+    test-unit (3.6.2)
       power_assert
-    tzinfo (2.0.5)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    webmock (3.23.0)
+    webmock (3.24.0)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
+  ruby
   x86_64-darwin-21
   x86_64-darwin-23
   x86_64-linux
@@ -78,4 +88,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   2.4.21
+   2.4.22

config/config.test.rb

@@ -1,14 +1,34 @@
-# config.rb is required for testing
-# unit test makes calls to bioportal api so it needs a valid API key which can
-# be set via ENV variable UT_APIKEY
-abort('UT_APIKEY env variable is not set. Canceling tests') unless ENV.include?('UT_APIKEY')
-abort('UT_APIKEY env variable is set to an empty value. Canceling tests') unless ENV['UT_APIKEY'].size > 5
 $API_CLIENT_INVALIDATE_CACHE = false
 $DEBUG_API_CLIENT = false
+
 LinkedData::Client.config do |config|
-  config.rest_url   = 'https://data.bioontology.org'
+  config.rest_url = 'https://data.bioontology.org/'
   config.apikey = '8b5b7825-538d-40e0-9e9e-5ab9274a9aeb'
   config.links_attr = 'links'
   config.cache = true
   config.debug_client = false
+  config.debug_client_keys = []
+  config.federated_portals = {
+    bioportal: {
+      api: 'https://data.agroportal.lirmm.fr/',
+      apikey: '1de0a270-29c5-4dda-b043-7c3580628cd5',
+      color: '#234979',
+    },
+    ecoportal: {
+      api: 'https://data.ecoportal.lifewatch.eu/',
+      apikey: "43a437ba-a437-4bf0-affd-ab520e584719",
+      color: '#0f4e8a',
+    },
+    # earthportal: {
+    #   api: 'https://earthportal.eu:8443/',
+    #   apikey: "c9147279-954f-41bd-b068-da9b0c441288",
+    #   color: '#1e2251',
+    # },
+    biodivportal: {
+      api: 'https://data.biodivportal.gfbio.dev/',
+      apikey: "47a57aa3-7b54-4f34-b695-dbb5f5b7363e",
+      color: '#1e2251',
+    }
+  }
+
 end

lib/ontologies_api_client.rb

@@ -1,6 +1,7 @@
 require 'oj'
 require 'multi_json'
 require 'spawnling'
+require 'request_store'
 
 require_relative 'ontologies_api_client/config'
 require_relative 'ontologies_api_client/http'

lib/ontologies_api_client/analytics.rb

@@ -1,6 +1,9 @@
+require_relative 'request_federation'
+
 module LinkedData::Client
   class Analytics
     HTTP = LinkedData::Client::HTTP
+    include LinkedData::Client::RequestFederation
 
     attr_accessor :onts, :date
 
@@ -10,26 +13,40 @@ def self.all(params = {})
 
     def self.last_month
       data = self.new
-      data.date = last_month = DateTime.now - 1.month
+      last_month = DateTime.now.prev_month
       year_num = last_month.year
       month_num = last_month.month
-      analytics = get(:analytics, {year: year_num, month: month_num}).to_h
-      analytics.delete(:links)
-      analytics.delete(:context)
+      params = { year: year_num, month: month_num }
+
+      responses = federated_get(params) do |url|
+        "#{url}/analytics"
+      end
+
+      portals = request_portals
       onts = []
-      analytics.keys.each do |ont|
-        views = analytics[ont][:"#{year_num}"][:"#{month_num}"]
-        onts << {ont: ont, views: views}
+      responses.each_with_index do |portal_views, index|
+        next nil if portal_views&.errors
+
+        portal_views = portal_views.to_h
+
+        url = portals[index].url_prefix.to_s.chomp('/')
+        portal_views.delete(:links)
+        portal_views.delete(:context)
+        portal_views.keys.map do |ont|
+          views = portal_views[ont][:"#{year_num}"][:"#{month_num}"]
+          onts << { ont: "#{url}/ontologies/#{ont}", views: views }
+        end
       end
-      data.onts = onts
+
+      data.onts = onts.flatten.compact
       data
     end
 
     private
 
     def self.get(path, params = {})
       path = path.to_s
-      path = "/"+path unless path.start_with?("/")
+      path = "/" + path unless path.start_with?("/")
       HTTP.get(path, params)
     end
 

lib/ontologies_api_client/base.rb

@@ -132,11 +132,16 @@ def create_attributes(attributes)
           attr_exists = self.public_methods(false).include?(attr)
           unless attr_exists
             self.class.class_eval do
-              define_method attr.to_sym do
-                instance_variable_get("@#{attr}")
+              unless method_defined?(attr.to_sym)
+                define_method attr.to_sym do
+                  instance_variable_get("@#{attr}")
+                end
               end
-              define_method "#{attr}=" do |val|
-                instance_variable_set("@#{attr}", val)
+
+              unless method_defined?("#{attr}=".to_sym)
+                define_method "#{attr}=" do |val|
+                  instance_variable_set("@#{attr}", val)
+                end
               end
             end
           end

lib/ontologies_api_client/collection.rb

@@ -1,11 +1,15 @@
 require_relative 'config'
 require_relative 'http'
+require_relative 'request_federation'
+require 'parallel'
 
 module LinkedData
   module Client
     module Collection
 
+
       def self.included(base)
+        base.include LinkedData::Client::RequestFederation
         base.extend(ClassMethods)
       end
 
@@ -24,8 +28,8 @@ def method_missing(meth, *args, &block)
 
         ##
         # Get all top-level links for the API
-        def top_level_links
-          @top_level_links||= HTTP.get(LinkedData::Client.settings.rest_url)
+        def top_level_links(link = LinkedData::Client.settings.rest_url)
+          HTTP.get(link)
         end
 
         ##
@@ -36,11 +40,14 @@ def uri_from_context(object, media_type)
           end
         end
 
+
         ##
         # Get the first collection of resources for a given type
         def entry_point(media_type, params = {})
-          params = {include: @include_attrs}.merge(params)
-          HTTP.get(uri_from_context(top_level_links, media_type), params)
+          params = { include: @include_attrs, display_links: false, display_context: false}.merge(params)
+          federated_get(params) do |url|
+            uri_from_context(top_level_links(url), media_type) rescue nil
+          end
         end
 
         ##

lib/ontologies_api_client/config.rb

@@ -37,21 +37,38 @@ def config(&block)
 
     def config_connection(options = {})
       return if @settings_run_connection
-      store = options[:cache_store]
-      @settings.conn = Faraday.new(@settings.rest_url) do |faraday|
+      store = options[:cache_store] || ActiveSupport::Cache::MemoryStore.new
+      @settings.conn = faraday_connection(@settings.rest_url, @settings.apikey, store, current_portal: true)
+      @settings.federated_conn = @settings.federated_portals.map do |portal_name, portal_info|
+        [portal_name, faraday_connection(portal_info[:api], portal_info[:apikey], store)]
+      end.to_h
+
+      @settings_run_connection = true
+    end
+
+    def connection_configured?
+      @settings_run_connection
+    end
+
+    private
+    def faraday_connection(url, apikey, store, current_portal: false)
+      Faraday.new(url.to_s.chomp('/')) do |faraday|
+
         if @settings.enable_long_request_log
           require_relative 'middleware/faraday-long-requests'
           faraday.use :long_requests
         end
 
-        require_relative 'middleware/faraday-user-apikey'
-        faraday.use :user_apikey
+        if current_portal
+          require_relative 'middleware/faraday-user-apikey'
+          faraday.use :user_apikey
 
-        require_relative 'middleware/faraday-slices'
-        faraday.use :ncbo_slices
+          require_relative 'middleware/faraday-slices'
+          faraday.use :ncbo_slices
 
-        require_relative 'middleware/faraday-last-updated'
-        faraday.use :last_updated
+          require_relative 'middleware/faraday-last-updated'
+          faraday.use :last_updated
+        end
 
         if @settings.cache
           begin
@@ -69,15 +86,10 @@ def config_connection(options = {})
         faraday.adapter :excon
         faraday.headers = {
           "Accept" => "application/json",
-          "Authorization" => "apikey token=#{@settings.apikey}",
+          "Authorization" => "apikey token=#{apikey}",
           "User-Agent" => "NCBO API Ruby Client v0.1.0"
         }
       end
-      @settings_run_connection = true
-    end
-
-    def connection_configured?
-      @settings_run_connection
     end
   end
 end

lib/ontologies_api_client/http.rb

@@ -3,6 +3,7 @@
 require 'digest'
 require 'ostruct'
 require 'benchmark'
+require 'active_support/cache'
 ##
 # This monkeypatch makes OpenStruct act like Struct objects
 class OpenStruct
@@ -48,30 +49,35 @@ def self.conn
             rails = Kernel.const_get("Rails")
             store = rails.cache if rails.cache
           end
-          LinkedData::Client.config_connection(cache_store: store)
+          LinkedData::Client.config_connection(cache_store: store || ActiveSupport::Cache::MemoryStore.new)
         end
         LinkedData::Client.settings.conn
       end
 
+      def self.federated_conn
+        LinkedData::Client.settings.federated_conn
+      end
+
       def self.get(path, params = {}, options = {})
         headers = options[:headers] || {}
         raw = options[:raw] || false # return the unparsed body of the request
         params = params.delete_if { |k, v| v == nil || v.to_s.empty? }
         params[:ncbo_cache_buster] = Time.now.to_f if raw # raw requests don't get cached to ensure body is available
         invalidate_cache = params.delete(:invalidate_cache) || $API_CLIENT_INVALIDATE_CACHE || false
+        connection = options[:connection] || conn
         begin
           begin
             response = nil
             time = Benchmark.realtime do
-              response = conn.get do |req|
+              response = connection.get do |req|
                 req.url path
                 req.params = params.dup
                 req.options[:timeout] = 60
                 req.headers.merge(headers)
                 req.headers[:invalidate_cache] = invalidate_cache
               end
             end
-            puts "Getting: #{path} with #{params} (#{time}s)" if $DEBUG_API_CLIENT
+            puts "Getting: #{path} with #{params} (t: #{time}s - cache: #{response.headers["X-Rack-Cache"]})" if $DEBUG_API_CLIENT
           rescue Exception => e
             params = Faraday::Utils.build_query(params)
             path << "?" unless params.empty? || path.include?("?")