Merge remote-tracking branch 'upstream/master' into merge-to-upstream…

…-v5.26.0

.dockerignore

@@ -1,5 +1,6 @@
 # Git
-#.git
+.git
+.github
 .gitignore
 # Logs
 log/*
@@ -8,3 +9,9 @@ tmp/*
 # Editor temp files
 *.swp
 *.swo
+coverage
+create_permissions.log
+# Ignore generated test data
+test/data/dictionary.txt
+test/data/ontology_files/repo/**/*
+test/data/tmp/*

.github/workflows/docker-image.yml

@@ -0,0 +1,42 @@
+name: Docker Image CI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: bioportal/ncbo_cron
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            RUBY_VERSION=2.7
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/ruby-unit-tests.yml

@@ -7,18 +7,17 @@ on:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
-        backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
+        backend: ['ncbo_cron', 'ncbo_cron-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
     - name: copy config.rb file from template
       run:  cp config/config.test.rb config/config.rb
     - name: Build docker-compose
-      working-directory: ./test
       run: docker-compose build
     - name: Run unit tests
-      working-directory: ./test
       run: |
         ci_env=`bash <(curl -s https://codecov.io/env)`
         docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v'

.gitignore

@@ -5,6 +5,8 @@ config/config.rb
 config/appliance.rb
 config/config_*.rb
 config/*.p12
+config/*.json
+data/
 projectFilesBackup/
 .ruby-version
 repo*

Dockerfile

@@ -20,4 +20,6 @@ ENV BUNDLE_PATH=/srv/ontoportal/bundle
 RUN bundle install
 
 COPY . /srv/ontoportal/ncbo_cron
+RUN cp /srv/ontoportal/ncbo_cron/config/config.rb.sample /srv/ontoportal/ncbo_cron/config/config.rb
+
 CMD ["/bin/bash"]

Gemfile

@@ -2,13 +2,11 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem 'faraday', '~> 1.9'
 gem 'ffi'
-gem "google-apis-analytics_v3"
+gem 'google-analytics-data'
 gem 'mail', '2.6.6'
-gem 'minitest', '< 5.0'
 gem 'multi_json'
-gem 'oj', '~> 2.0'
+gem 'oj', '~> 3.0'
 gem 'parseconfig'
 gem 'pony'
 gem 'pry'
@@ -28,8 +26,8 @@ gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'master
 # Testing
 group :test do
   gem 'email_spec'
+  gem 'minitest', '< 5.0'
   gem 'simplecov'
   gem 'simplecov-cobertura' # for codecov.io
   gem 'test-unit-minitest'
 end
-

Gemfile.lock

@@ -60,7 +60,7 @@ PATH
     ncbo_cron (0.0.1)
       dante
       goo
-      google-apis-analytics_v3
+      google-analytics-data
       mlanett-redis-lock
       multi_json
       ncbo_annotator
@@ -83,7 +83,6 @@ GEM
     connection_pool (2.4.1)
     cube-ruby (0.0.3)
     dante (0.2.0)
-    declarative (0.0.20)
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
@@ -103,17 +102,9 @@ GEM
       faraday-rack (~> 1.0)
       faraday-retry (~> 1.0)
       ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
+    faraday-net_http (3.0.2)
+    faraday-retry (2.2.0)
+      faraday (~> 2.0)
     ffi (1.15.5)
     google-apis-analytics_v3 (0.13.0)
       google-apis-core (>= 0.11.0, < 2.a)
@@ -129,15 +120,22 @@ GEM
     googleauth (1.7.0)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
-      memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
+    grpc (1.58.0)
+      google-protobuf (~> 3.23)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.58.0-x86_64-darwin)
+      google-protobuf (~> 3.23)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.58.0-x86_64-linux)
+      google-protobuf (~> 3.23)
+      googleapis-common-protos-types (~> 1.0)
     htmlentities (4.3.4)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
-    httpclient (2.8.3)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
@@ -151,9 +149,8 @@ GEM
       systemu (~> 2.6.5)
     mail (2.6.6)
       mime-types (>= 1.16, < 4)
-    memoist (0.16.2)
     method_source (1.0.0)
-    mime-types (3.4.1)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0218.1)
     mini_mime (1.1.2)
@@ -164,7 +161,7 @@ GEM
     multipart-post (2.3.0)
     net-http-persistent (2.9.4)
     netrc (0.11.0)
-    oj (2.18.5)
+    oj (3.16.1)
     omni_logger (0.1.4)
       logger
     os (1.1.4)
@@ -196,8 +193,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    retriable (3.1.2)
-    rexml (3.2.5)
+    rexml (3.2.6)
     rsolr (2.5.0)
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
@@ -206,7 +202,7 @@ GEM
     rubyzip (2.3.2)
     rufus-scheduler (2.0.24)
       tzinfo (>= 0.3.22)
-    signet (0.17.0)
+    signet (0.18.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
@@ -228,7 +224,6 @@ GEM
     trailblazer-option (0.1.2)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    uber (0.1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.2)
@@ -242,16 +237,15 @@ PLATFORMS
 DEPENDENCIES
   cube-ruby
   email_spec
-  faraday (~> 1.9)
   ffi
   goo!
-  google-apis-analytics_v3
+  google-analytics-data
   mail (= 2.6.6)
   minitest (< 5.0)
   multi_json
   ncbo_annotator!
   ncbo_cron!
-  oj (~> 2.0)
+  oj (~> 3.0)
   ontologies_linked_data!
   parseconfig
   pony

bin/ncbo_cron

@@ -111,19 +111,9 @@ opt_parser = OptionParser.new do |opts|
   opts.on("--disable-update-check", "disable check for updated version of Ontoportal (for VMs)", "(default: #{options[:enable_update_check]})") do |v|
     options[:enable_update_check] = false
   end
-
-
-
-
-  opts.on("--disable-dictionary-generation", "disable mgrep dictionary generation job", "(default: #{options[:enable_dictionary_generation]})") do |v|
-    options[:enable_dictionary_generation] = false
+  opts.on("--enable-dictionary-generation-cron-job", "ENABLE mgrep dictionary generation JOB and DISABLE dictionary generation during ontology processing. If this is not passed in, dictionary is generated every time an ontology is processed.", "(default: Dictionary is generated on every ontology processing, CRON job is DISABLED)") do |v|
+    options[:enable_dictionary_generation_cron_job] = true
   end
-
-
-
-
-
-
   opts.on("--disable-obofoundry_sync", "disable OBO Foundry synchronization report", "(default: #{options[:enable_obofoundry_sync]})") do |v|
     options[:enable_obofoundry_sync] = false
   end
@@ -166,18 +156,10 @@ opt_parser = OptionParser.new do |opts|
   opts.on("--obofoundry_sync SCHED", String, "cron schedule to run OBO Foundry synchronization report", "(default: #{options[:cron_obofoundry_sync]})") do |c|
     options[:cron_obofoundry_sync] = c
   end
-
-
-
-
-  opts.on("--dictionary-generation SCHED", String, "cron schedule to run mgrep dictionary generation job", "(default: #{options[:cron_dictionary_generation]})") do |c|
-    options[:cron_dictionary_generation] = c
+  opts.on("--dictionary-generation-cron-job SCHED", String, "cron schedule to run mgrep dictionary generation job (if enabled)", "(default: #{options[:cron_dictionary_generation_cron_job]})") do |c|
+    options[:cron_dictionary_generation_cron_job] = c
   end
 
-
-
-
-
   # Display the help screen, all programs are assumed to have this option.
   opts.on_tail('--help', 'Display this screen') do
     puts opts
@@ -507,49 +489,27 @@ runner.execute do |opts|
     end
   end
 
-
-
-
-
-
-
-
-  # temporary job to generate mgrep dictionary file
+  # optional job to generate mgrep dictionary file
   # separate from ontology processing due to
   # https://github.com/ncbo/ncbo_cron/issues/45
-
-  if options[:enable_dictionary_generation]
+  if options[:enable_dictionary_generation_cron_job]
     dictionary_generation_thread = Thread.new do
       dictionary_generation_options = options.dup
-      dictionary_generation_options[:job_name] = "ncbo_cron_dictionary_generation"
+      dictionary_generation_options[:job_name] = "ncbo_cron_dictionary_generation_cron_job"
       dictionary_generation_options[:scheduler_type] = :cron
-      dictionary_generation_options[:cron_schedule] = dictionary_generation_options[:cron_dictionary_generation]
-      logger.info "Setting up mgrep dictionary generation job with #{dictionary_generation_options[:cron_dictionary_generation]}"; logger.flush
+      dictionary_generation_options[:cron_schedule] = dictionary_generation_options[:cron_dictionary_generation_cron_job]
+      logger.info "Setting up mgrep dictionary generation job with #{dictionary_generation_options[:cron_dictionary_generation_cron_job]}"; logger.flush
       NcboCron::Scheduler.scheduled_locking_job(dictionary_generation_options) do
-        logger.info "Starting mgrep dictionary generation..."; logger.flush
+        logger.info "Starting mgrep dictionary generation CRON job..."; logger.flush
         t0 = Time.now
         annotator = Annotator::Models::NcboAnnotator.new
         annotator.generate_dictionary_file()
-        logger.info "mgrep dictionary generation job completed in #{Time.now - t0} sec."; logger.flush
-        logger.info "Finished mgrep dictionary generation"; logger.flush
+        logger.info "mgrep dictionary generation CRON job completed in #{Time.now - t0} sec."; logger.flush
+        logger.info "Finished mgrep dictionary generation CRON job"; logger.flush
       end
     end
   end
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
   # Print running child processes
   require 'sys/proctable'
   at_exit do
@@ -573,12 +533,5 @@ runner.execute do |opts|
   mapping_counts_thread.join if mapping_counts_thread
   update_check_thread.join if update_check_thread
   obofoundry_sync_thread.join if obofoundry_sync_thread
-
-
-
-
   dictionary_generation_thread.join if dictionary_generation_thread
-
-
-
 end

bin/ncbo_ontology_annotate_generate_cache

@@ -49,7 +49,7 @@ opt_parser = OptionParser.new do |opts|
     options[:generate_dictionary] = true
   end
 
-  options[:logfile] = "logs/annotator_cache.log"
+  options[:logfile] = STDOUT
   opts.on('-l', '--logfile FILE', "Write log to FILE (default is 'logs/annotator_cache.log').") do |filename|
     options[:logfile] = filename
   end