Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into biodivportal-align-…
Browse files Browse the repository at this point in the history
…2-9-1
  • Loading branch information
syphax-bouazzouni committed Aug 13, 2024
2 parents d1a1603 + ffe7cb6 commit c466e92
Show file tree
Hide file tree
Showing 84 changed files with 4,175 additions and 848 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ tmp/*
# Editor temp files
*.swp
*.swo
test/solr
4 changes: 4 additions & 0 deletions .env.sample
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
API_URL=http://localhost:9393
ONTOLOGIES_LINKED_DATA_PATH=
GOO_PATH=
SPARQL_CLIENT_PATH=
123 changes: 65 additions & 58 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -1,85 +1,92 @@
# Workflow for deploying ontologies_api to stage/prod systems via capistrano.
# This workflow runs after a successeful execution of the unit test workflow and it
# can also be triggered manually.
# Workflow to deploy OntoPortal API to stage/prod systems
#
# Required github secrets:
#
# CONFIG_REPO - github repo containing config and customizations for the API. Format 'author/private_config_repo'
# CONFIG_REPO - github repo containing config and customizations for API. Format 'author/private_config_repo'
# it is used for getting capistrano deployment configuration for stages on the github actions runner and
# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the remote servers for pulling configs.
# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the API hosts for pulling configs.
#
# GH_PAT - github Personal Access Token for accessing PRIVATE_CONFIG_REPO
# GH_PAT - github Personal Access Token for accessing private config repo
#
# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if app servers are hosted on private network.
# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if API nodes live on private network.
# SSH_JUMPHOST_USER - username to use to connect to the ssh jump/proxy.
#
# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/deploy_id_rsa_enc (see miloserdow/capistrano-deploy)
# this SSH key is used for accessing jump host, UI nodes, and private github repo.
# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/
# this SSH key is used for accessing jump host, API nodes, and private github repo.

name: Capistrano Deployment
# Controls when the action will run.
on:
# Trigger deployment to staging after unit test action completes
workflow_run:
workflows: ["Ruby Unit Tests"]
types:
- completed
branches: [master, develop]
push:
branches:
- stage
- test
# Allows running this workflow manually from the Actions tab
workflow_dispatch:
branches: [master, develop]
inputs:
BRANCH:
description: 'Branch/tag to deploy'
default: develop
description: "Branch/tag to deploy"
options:
- stage
- test
- master
default: stage
required: true
environment:
description: 'target environment to deploy to'
description: "target environment to deploy to"
type: choice
options:
- staging
- production
default: staging

- agroportal
- test
default: stage
jobs:
deploy:
runs-on: ubuntu-latest
# run deployment only if "Ruby Unit Tests" workflow completes sucessefully or when manually triggered
if: ${{ (github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch') }}
env:
BUNDLE_WITHOUT: default #install gems required primarily for the deployment in order to speed this workflow
BUNDLE_WITHOUT: default #install gems required primarely for deployment in order to speed up workflow
PRIVATE_CONFIG_REPO: ${{ format('[email protected]:{0}.git', secrets.CONFIG_REPO) }}
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- name: set branch/tag and environment to deploy from inputs
run: |
# workflow_dispatch default input doesn't get set on push so we need to set defaults
# via shell parameter expansion
# https://dev.to/mrmike/github-action-handling-input-default-value-5f2g
USER_INPUT_BRANCH=${{ inputs.branch }}
echo "BRANCH=${USER_INPUT_BRANCH:-develop}" >> $GITHUB_ENV
USER_INPUT_ENVIRONMENT=${{ inputs.environment }}
echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3
- uses: ruby/setup-ruby@v1
with:
ruby-version: 2.7.6 # Not needed with a .ruby-version file
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- name: get-deployment-config
uses: actions/checkout@v3
with:
repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings
token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT
path: deploy_config
- name: copy-deployment-config
run: cp -r deploy_config/ontologies_api/* .
# add ssh hostkey so that capistrano doesn't complain
- name: Add jumphost's hostkey to Known Hosts
run: |
mkdir -p ~/.ssh
ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts
shell: bash
- uses: miloserdow/capistrano-deploy@master
with:
target: ${{ env.TARGET }} # which environment to deploy
deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project
- name: set branch/tag and environment to deploy from inputs
run: |
# workflow_dispatch default input doesn't get set on push so we need to set defaults
# via shell parameter expansion
# https://dev.to/mrmike/github-action-handling-input-default-value-5f2g
USER_INPUT_BRANCH=${{ inputs.branch }}
echo "BRANCH=${USER_INPUT_BRANCH:github.head_ref:-master}" >> $GITHUB_ENV
USER_INPUT_ENVIRONMENT=${{ inputs.environment }}
echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV
CONFIG_REPO=${{ secrets.CONFIG_REPO }}
GH_PAT=${{ secrets.GH_PAT }}
echo "PRIVATE_CONFIG_REPO=https://${GH_PAT}@github.com/${CONFIG_REPO}" >> $GITHUB_ENV
echo "SSH_JUMPHOST=${{ secrets.SSH_JUMPHOST }}" >> $GITHUB_ENV
echo "SSH_JUMPHOST_USER=${{ secrets.SSH_JUMPHOST_USER }}" >> $GITHUB_ENV
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3
- uses: ruby/setup-ruby@v1
with:
ruby-version: 2.7.6 # Not needed with a .ruby-version file
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- name: get-deployment-config
uses: actions/checkout@v3
with:
repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings
token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT
path: deploy_config
- name: copy-deployment-config
run: cp -r deploy_config/ontologies_api/${{ inputs.environment }}/* .
# add ssh hostkey so that capistrano doesn't complain
- name: Add jumphost's hostkey to Known Hosts
run: |
mkdir -p ~/.ssh
echo "${{ secrets.SSH_JUMPHOST }}"
ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts
shell: bash
- uses: miloserdow/capistrano-deploy@master
with:
target: ${{ env.TARGET }} # which environment to deploy
deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project
56 changes: 56 additions & 0 deletions .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
name: Docker branch Images build

on:
push:
branches:
- master
- development
- stage
- test
release:
types: [ published ]
jobs:
push_to_registry:
name: Push Docker branch image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2

- name: Log in to Docker Hub
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4
with:
images: |
agroportal/ontologies_api
ghcr.io/${{ github.repository }}
- name: Build and push Docker image
uses: docker/build-push-action@v4
with:
context: .
platforms: linux/amd64,linux/arm64
build-args: |
RUBY_VERSION=2.7.8
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
45 changes: 29 additions & 16 deletions .github/workflows/ruby-unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,35 @@ on:
jobs:
test:
strategy:
fail-fast: false
matrix:
backend: ['api'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend
goo-slice: [ '20', '100', '500' ]
ruby-version: [ '2.7' ]
triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build docker-compose
run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step
- name: Run unit tests
# unit tests are run inside a container
# http://docs.codecov.io/docs/testing-with-docker
run: |
ci_env=`bash <(curl -s https://codecov.io/env)`
docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v'
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v3
with:
flags: unittests
verbose: true
fail_ci_if_error: false # optional (default = false)
- uses: actions/checkout@v3
- name: Install Dependencies
run: sudo apt-get update && sudo apt-get -y install raptor2-utils
- name: Set up JDK 11
uses: actions/setup-java@v2
with:
java-version: '11'
distribution: 'adopt'
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- name: Run unit tests
# unit tests are run inside a container
# http://docs.codecov.io/docs/testing-with-docker
run: |
ci_env=`bash <(curl -s https://codecov.io/env)`
GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v3
with:
flags: unittests
verbose: true
fail_ci_if_error: false # optional (default = false)
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ config/environments/*
!config/environments/biodiv-denbi-config.rb.sample

#ignore capistrano deployment
config/deploy/*
config/*.p12

# Ignore generated test data
Expand Down Expand Up @@ -77,3 +76,5 @@ ontologies_api.iml
.rubocop.yml
.solargraph.yml
.vscode

.env
5 changes: 4 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
openjdk-11-jre-headless \
raptor2-utils \
wait-for-it \
libraptor2-dev \
&& rm -rf /var/lib/apt/lists/*

RUN mkdir -p /srv/ontoportal/ontologies_api
Expand All @@ -15,12 +16,14 @@ COPY Gemfile* /srv/ontoportal/ontologies_api/

WORKDIR /srv/ontoportal/ontologies_api

RUN gem update --system
RUN gem update --system 3.4.22 # the 3.4.22 can be removed if we support Ruby version > 3.0
RUN gem install bundler
ENV BUNDLE_PATH=/srv/ontoportal/bundle
RUN bundle install

COPY . /srv/ontoportal/ontologies_api
RUN cp /srv/ontoportal/ontologies_api/config/environments/config.rb.sample /srv/ontoportal/ontologies_api/config/environments/development.rb
RUN cp /srv/ontoportal/ontologies_api/config/environments/config.rb.sample /srv/ontoportal/ontologies_api/config/environments/production.rb

EXPOSE 9393
CMD ["bundle", "exec", "rackup", "-p", "9393", "--host", "0.0.0.0"]
18 changes: 12 additions & 6 deletions Gemfile
Original file line number Diff line number Diff line change
@@ -1,24 +1,28 @@
source 'https://rubygems.org'

gem 'activesupport', '~> 3.0'
gem 'activesupport', '~> 3.2'
# see https://github.com/ncbo/ontologies_api/issues/69
gem 'bigdecimal', '1.4.2'
gem 'faraday', '~> 1.9'
gem 'json-schema', '~> 2.0'
gem 'multi_json', '~> 1.0'
gem 'oj', '~> 2.0'
gem 'oj'
gem 'parseconfig'
gem 'rack'
gem 'rake', '~> 10.0'
gem 'sinatra', '~> 1.0'
gem 'sinatra-advanced-routes'
gem 'sinatra-contrib', '~> 1.0'
gem 'request_store'
gem 'parallel'
gem 'json-ld'


# Rack middleware
gem 'ffi'
gem 'rack-accept', '~> 0.4'
gem 'rack-attack', '~> 6.6.1', require: 'rack/attack'
gem 'rack-cache', '~> 1.0'
gem 'rack-cache', '~> 1.13.0'
gem 'rack-cors', require: 'rack/cors'
# GitHub dependency can be removed when https://github.com/niko/rack-post-body-to-params/pull/6 is merged and released
gem 'rack-post-body-to-params', github: 'palexander/rack-post-body-to-params', branch: 'multipart_support'
Expand All @@ -27,7 +31,7 @@ gem 'redis-rack-cache', '~> 2.0'

# Data access (caching)
gem 'redis'
gem 'redis-activesupport'
gem 'redis-store', '~>1.10'

# Monitoring
gem 'cube-ruby', require: 'cube'
Expand All @@ -43,10 +47,10 @@ gem 'redcarpet'

# NCBO gems (can be from a local dev path or from rubygems/git)
gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'master'
gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master'
gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development'
gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master'
gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master'
gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'
gem 'ontologies_linked_data', git: 'https://github.com/BiodivPortal/ontologies_linked_data.git', branch: 'master'

group :development do
Expand All @@ -66,6 +70,7 @@ group :development do
gem 'reek', require: false
end


group :profiling do
gem 'rack-mini-profiler'
end
Expand All @@ -76,4 +81,5 @@ group :test do
gem 'rack-test'
gem 'simplecov', require: false
gem 'simplecov-cobertura' # for codecov.io
gem 'webmock', '~> 3.19.1'
end
Loading

0 comments on commit c466e92

Please sign in to comment.