Feature: implement federation of collections calls

Gemfile.lock

@@ -2,20 +2,22 @@ PATH
   remote: .
   specs:
     ontologies_api_client (2.2.0)
-      activesupport
+      activesupport (~> 7.0.4)
       excon
       faraday
       faraday-excon (~> 2.0.0)
       faraday-multipart
       lz4-ruby
       multi_json
       oj
+      parallel
+      request_store
       spawnling (= 2.1.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.4)
+    activesupport (7.0.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -24,11 +26,11 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     bigdecimal (3.1.7)
     coderay (1.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.3)
     crack (1.0.0)
       bigdecimal
       rexml
-    excon (0.95.0)
+    excon (0.110.0)
     faraday (2.0.1)
       faraday-net_http (~> 2.0)
       ruby2_keywords (>= 0.0.4)
@@ -39,26 +41,31 @@ GEM
       multipart-post (~> 2)
     faraday-net_http (2.1.0)
     hashdiff (1.1.0)
-    i18n (1.12.0)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
     lz4-ruby (0.3.3)
-    method_source (1.0.0)
-    minitest (5.16.3)
+    method_source (1.1.0)
+    minitest (5.22.3)
     multi_json (1.15.0)
-    multipart-post (2.2.3)
-    oj (3.13.23)
-    power_assert (2.0.2)
-    pry (0.14.1)
+    multipart-post (2.4.0)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
+    parallel (1.24.0)
+    power_assert (2.0.3)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.5)
-    rake (13.0.6)
+    rack (3.0.10)
+    rake (13.2.1)
+    request_store (1.7.0)
+      rack (>= 1.4)
     rexml (3.2.6)
     ruby2_keywords (0.0.5)
     spawnling (2.1.5)
-    test-unit (3.5.7)
+    test-unit (3.6.2)
       power_assert
-    tzinfo (2.0.5)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     webmock (3.23.0)
       addressable (>= 2.8.0)

config/config.test.rb

@@ -1,14 +1,33 @@
-# config.rb is required for testing
-# unit test makes calls to bioportal api so it needs a valid API key which can
-# be set via ENV variable UT_APIKEY
-abort('UT_APIKEY env variable is not set. Canceling tests') unless ENV.include?('UT_APIKEY')
-abort('UT_APIKEY env variable is set to an empty value. Canceling tests') unless ENV['UT_APIKEY'].size > 5
 $API_CLIENT_INVALIDATE_CACHE = false
 $DEBUG_API_CLIENT = false
+
 LinkedData::Client.config do |config|
-  config.rest_url   = 'https://data.bioontology.org'
+  config.rest_url = 'https://data.bioontology.org/'
   config.apikey = '8b5b7825-538d-40e0-9e9e-5ab9274a9aeb'
   config.links_attr = 'links'
   config.cache = true
   config.debug_client = false
+  config.debug_client_keys = []
+  config.federated_portals = {
+    bioportal: {
+      api: 'https://data.agroportal.lirmm.fr/',
+      apikey: '1de0a270-29c5-4dda-b043-7c3580628cd5',
+      color: '#234979',
+    },
+    ecoportal: {
+      api: 'https://data.ecoportal.lifewatch.eu/',
+      apikey: "43a437ba-a437-4bf0-affd-ab520e584719",
+      color: '#0f4e8a',
+    },
+    # earthportal: {
+    #   api: 'https://earthportal.eu:8443/',
+    #   apikey: "c9147279-954f-41bd-b068-da9b0c441288",
+    #   color: '#1e2251',
+    # },
+    biodivportal: {
+      api: 'https://data.biodivportal.gfbio.org/',
+      apikey: "47a57aa3-7b54-4f34-b695-dbb5f5b7363e",
+      color: '#1e2251',
+    }
+  }
 end

lib/ontologies_api_client.rb

@@ -1,6 +1,7 @@
 require 'oj'
 require 'multi_json'
 require 'spawnling'
+require 'request_store'
 
 require_relative 'ontologies_api_client/config'
 require_relative 'ontologies_api_client/http'

lib/ontologies_api_client/analytics.rb

@@ -1,6 +1,9 @@
+require_relative 'request_federation'
+
 module LinkedData::Client
   class Analytics
     HTTP = LinkedData::Client::HTTP
+    include LinkedData::Client::RequestFederation
 
     attr_accessor :onts, :date
 
@@ -10,26 +13,40 @@ def self.all(params = {})
 
     def self.last_month
       data = self.new
-      data.date = last_month = DateTime.now - 1.month
+      last_month = DateTime.now.prev_month
       year_num = last_month.year
       month_num = last_month.month
-      analytics = get(:analytics, {year: year_num, month: month_num}).to_h
-      analytics.delete(:links)
-      analytics.delete(:context)
+      params = { year: year_num, month: month_num }
+
+      responses = federated_get(params) do |url|
+        "#{url}/analytics"
+      end
+
+      portals = request_portals
       onts = []
-      analytics.keys.each do |ont|
-        views = analytics[ont][:"#{year_num}"][:"#{month_num}"]
-        onts << {ont: ont, views: views}
+      responses.each_with_index do |portal_views, index|
+        next nil if portal_views&.errors
+
+        portal_views = portal_views.to_h
+
+        url = portals[index].url_prefix.to_s.chomp('/')
+        portal_views.delete(:links)
+        portal_views.delete(:context)
+        portal_views.keys.map do |ont|
+          views = portal_views[ont][:"#{year_num}"][:"#{month_num}"]
+          onts << { ont: "#{url}/ontologies/#{ont}", views: views }
+        end
       end
-      data.onts = onts
+
+      data.onts = onts.flatten.compact
       data
     end
 
     private
 
     def self.get(path, params = {})
       path = path.to_s
-      path = "/"+path unless path.start_with?("/")
+      path = "/" + path unless path.start_with?("/")
       HTTP.get(path, params)
     end
 

lib/ontologies_api_client/base.rb

@@ -132,11 +132,16 @@ def create_attributes(attributes)
           attr_exists = self.public_methods(false).include?(attr)
           unless attr_exists
             self.class.class_eval do
-              define_method attr.to_sym do
-                instance_variable_get("@#{attr}")
+              unless method_defined?(attr.to_sym)
+                define_method attr.to_sym do
+                  instance_variable_get("@#{attr}")
+                end
               end
-              define_method "#{attr}=" do |val|
-                instance_variable_set("@#{attr}", val)
+
+              unless method_defined?("#{attr}=".to_sym)
+                define_method "#{attr}=" do |val|
+                  instance_variable_set("@#{attr}", val)
+                end
               end
             end
           end

lib/ontologies_api_client/collection.rb

@@ -1,11 +1,15 @@
 require_relative 'config'
 require_relative 'http'
+require_relative 'request_federation'
+require 'parallel'
 
 module LinkedData
   module Client
     module Collection
 
+
       def self.included(base)
+        base.include LinkedData::Client::RequestFederation
         base.extend(ClassMethods)
       end
 
@@ -24,8 +28,8 @@ def method_missing(meth, *args, &block)
 
         ##
         # Get all top-level links for the API
-        def top_level_links
-          @top_level_links||= HTTP.get(LinkedData::Client.settings.rest_url)
+        def top_level_links(link = LinkedData::Client.settings.rest_url)
+          HTTP.get(link)
         end
 
         ##
@@ -36,11 +40,14 @@ def uri_from_context(object, media_type)
           end
         end
 
+
         ##
         # Get the first collection of resources for a given type
         def entry_point(media_type, params = {})
-          params = {include: @include_attrs}.merge(params)
-          HTTP.get(uri_from_context(top_level_links, media_type), params)
+          params = { include: @include_attrs, display_links: false, display_context: false}.merge(params)
+          federated_get(params) do |url|
+            uri_from_context(top_level_links(url), media_type) rescue nil
+          end
         end
 
         ##

lib/ontologies_api_client/config.rb

@@ -37,8 +37,22 @@ def config(&block)
 
     def config_connection(options = {})
       return if @settings_run_connection
-      store = options[:cache_store]
-      @settings.conn = Faraday.new(@settings.rest_url) do |faraday|
+      store = options[:cache_store] || ActiveSupport::Cache::MemoryStore.new
+      @settings.conn = faraday_connection(@settings.rest_url, @settings.apikey, store)
+      @settings.federated_conn = @settings.federated_portals.map do |portal_name, portal_info|
+        [portal_name, faraday_connection(portal_info[:api], portal_info[:apikey], store)]
+      end.to_h
+
+      @settings_run_connection = true
+    end
+
+    def connection_configured?
+      @settings_run_connection
+    end
+
+    private
+    def faraday_connection(url, apikey, store)
+      Faraday.new(url.to_s.chomp('/')) do |faraday|
         if @settings.enable_long_request_log
           require_relative 'middleware/faraday-long-requests'
           faraday.use :long_requests
@@ -69,15 +83,10 @@ def config_connection(options = {})
         faraday.adapter :excon
         faraday.headers = {
           "Accept" => "application/json",
-          "Authorization" => "apikey token=#{@settings.apikey}",
+          "Authorization" => "apikey token=#{apikey}",
           "User-Agent" => "NCBO API Ruby Client v0.1.0"
         }
       end
-      @settings_run_connection = true
-    end
-
-    def connection_configured?
-      @settings_run_connection
     end
   end
 end

lib/ontologies_api_client/http.rb

@@ -3,6 +3,7 @@
 require 'digest'
 require 'ostruct'
 require 'benchmark'
+require 'active_support/cache'
 ##
 # This monkeypatch makes OpenStruct act like Struct objects
 class OpenStruct
@@ -48,30 +49,35 @@ def self.conn
             rails = Kernel.const_get("Rails")
             store = rails.cache if rails.cache
           end
-          LinkedData::Client.config_connection(cache_store: store)
+          LinkedData::Client.config_connection(cache_store: store || ActiveSupport::Cache::MemoryStore.new)
         end
         LinkedData::Client.settings.conn
       end
 
+      def self.federated_conn
+        LinkedData::Client.settings.federated_conn
+      end
+
       def self.get(path, params = {}, options = {})
         headers = options[:headers] || {}
         raw = options[:raw] || false # return the unparsed body of the request
         params = params.delete_if { |k, v| v == nil || v.to_s.empty? }
         params[:ncbo_cache_buster] = Time.now.to_f if raw # raw requests don't get cached to ensure body is available
         invalidate_cache = params.delete(:invalidate_cache) || $API_CLIENT_INVALIDATE_CACHE || false
+        connection = options[:connection] || conn
         begin
           begin
             response = nil
             time = Benchmark.realtime do
-              response = conn.get do |req|
+              response = connection.get do |req|
                 req.url path
                 req.params = params.dup
                 req.options[:timeout] = 60
                 req.headers.merge(headers)
                 req.headers[:invalidate_cache] = invalidate_cache
               end
             end
-            puts "Getting: #{path} with #{params} (#{time}s)" if $DEBUG_API_CLIENT
+            puts "Getting: #{path} with #{params} (t: #{time}s - cache: #{response.headers["X-Rack-Cache"]})" if $DEBUG_API_CLIENT
           rescue Exception => e
             params = Faraday::Utils.build_query(params)
             path << "?" unless params.empty? || path.include?("?")

lib/ontologies_api_client/middleware/faraday-object-cache.rb

@@ -1,4 +1,5 @@
 require 'digest/sha1'
+require 'active_support'
 require 'active_support/cache'
 require 'lz4-ruby'
 require_relative '../http'
@@ -70,7 +71,7 @@ def retrieve_cached_response(request_key)
       env = { status: 304 }
       cached_response = ObjectCacheResponse.new(env)
       cached_response.parsed_body = ld_obj
-      cached_response.env.response_headers = { "x-rack-cache" => 'hit' }
+      cached_response.env.response_headers = { "X-Rack-Cache" => 'hit' }
       cached_response
     end
 
@@ -88,7 +89,7 @@ def process_response(response_env, request_key)
 
       response = ObjectCacheResponse.new(response_env)
       response.parsed_body = ld_obj
-      response.env.response_headers["x-rack-cache"] = cache_state
+      response.env.response_headers["X-Rack-Cache"] = cache_state
       response
     end