Merge to master: AgroPortal release 2.8.0

.dockerignore

@@ -1,5 +1,6 @@
 # Git
-#.git
+.git
+.github
 .gitignore
 # Logs
 log/*
@@ -8,3 +9,9 @@ tmp/*
 # Editor temp files
 *.swp
 *.swo
+coverage
+create_permissions.log
+# Ignore generated test data
+test/data/dictionary.txt
+test/data/ontology_files/repo/**/*
+test/data/tmp/*

.github/workflows/docker-image.yml

@@ -0,0 +1,42 @@
+name: Docker Image CI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: bioportal/ncbo_cron
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            RUBY_VERSION=2.7
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/ruby-unit-tests.yml

@@ -7,25 +7,37 @@ on:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
-        backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
+        goo-slice: [ '20', '100', '500' ]
+        ruby-version: [ '2.7' ]
+        triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: copy config.rb file from template
-      run:  cp config/config.test.rb config/config.rb
-    - name: Build docker-compose
-      working-directory: ./test
-      run: docker-compose build
-    - name: Run unit tests
-      working-directory: ./test
-      run: |
-        ci_env=`bash <(curl -s https://codecov.io/env)`
-        docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v'
-    - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        flags: unittests
-        verbose: true
-        fail_ci_if_error: false # optional (default = false)
-
+      - uses: actions/checkout@v3
+      - name: create config.rb file
+        run: cp config/config.rb.sample config/config.rb
+      - name: Install Dependencies
+        run: sudo apt-get update && sudo apt-get -y install raptor2-utils
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run unit tests
+        # unit tests are run inside a container
+        # http://docs.codecov.io/docs/testing-with-docker
+        run: |
+          ci_env=`bash <(curl -s https://codecov.io/env)`
+          GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+          verbose: true
+          fail_ci_if_error: false # optional (default = false)

.gitignore

@@ -5,6 +5,8 @@ config/config.rb
 config/appliance.rb
 config/config_*.rb
 config/*.p12
+config/*.json
+data/
 projectFilesBackup/
 .ruby-version
 repo*

Dockerfile

@@ -14,10 +14,16 @@ COPY Gemfile* *.gemspec /srv/ontoportal/ncbo_cron/
 
 WORKDIR /srv/ontoportal/ncbo_cron
 
+# set rubygem and bundler to the last version supported by ruby 2.7
+# remove version after ruby v3 upgrade
+RUN gem update --system '3.4.22'
+RUN gem install bundler -v '2.4.22'
 RUN gem update --system
 RUN gem install bundler
 ENV BUNDLE_PATH=/srv/ontoportal/bundle
 RUN bundle install
 
 COPY . /srv/ontoportal/ncbo_cron
+RUN cp /srv/ontoportal/ncbo_cron/config/config.rb.sample /srv/ontoportal/ncbo_cron/config/config.rb
+
 CMD ["/bin/bash"]

Gemfile

@@ -2,44 +2,49 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem 'faraday', '~> 1.9'
-gem 'activesupport', '~> 3.2'
-gem 'ffi', '~> 1.15.5'
+gem 'ffi'
+
+# This is needed temporarily to pull the Google Universal Analytics (UA)
+# data and store it in a file. See (bin/import_google_ua_analytics_data)
+# The ability to pull this data from Google will cease on July 1, 2024
 gem "google-apis-analytics_v3"
+
+gem 'google-analytics-data'
 gem 'mail', '2.6.6'
-gem 'minitest', '< 5.0'
 gem 'multi_json'
-gem 'oj', '~> 2.0'
+gem 'oj', '~> 3.0'
 gem 'parseconfig'
 gem 'pony'
 gem 'pry'
 gem 'rake'
 gem 'redis'
 gem 'rest-client'
 gem 'sys-proctable'
-
+gem 'request_store'
+gem 'parallel'
+gem 'json-ld'
+gem 'activesupport', '~>  3.2.22'
 # Monitoring
 gem 'cube-ruby', require: 'cube'
 
-gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'ecoportal'
-gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
+gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development'
+gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'
+gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'development'
 gem 'ontologies_linked_data', git: 'https://github.com/lifewatch-eric/ontologies_linked_data.git', branch: 'master'
-gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'master'
 
 group :development do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
-  gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false
   gem 'capistrano', '~> 3', require: false
+  gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false
   gem 'capistrano-bundler', require: false
   gem 'capistrano-locally', require: false
   gem 'capistrano-rbenv', require: false
   gem 'ed25519', '>= 1.2', '< 2.0', require: false
 end
-
-
 # Testing
 group :test do
   gem 'email_spec'
+  gem 'minitest', '< 5.0'
   gem 'simplecov'
   gem 'simplecov-cobertura' # for codecov.io
   gem 'test-unit-minitest'