Merge branch 'master' into development

.github/workflows/deploy.yml

@@ -1,85 +1,92 @@
-# Workflow for deploying ontologies_api to stage/prod systems via capistrano.
-# This workflow runs after a successeful execution of the unit test workflow and it
-# can also be triggered manually.
+# Workflow to deploy OntoPortal API to stage/prod systems
 #
 # Required github secrets:
 #
-# CONFIG_REPO - github repo containing config and customizations for the API. Format 'author/private_config_repo'
+# CONFIG_REPO - github repo containing config and customizations for API. Format 'author/private_config_repo'
 # it is used for getting capistrano deployment configuration for stages on the github actions runner and
-# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the remote servers for pulling configs.
+# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the API hosts for pulling configs.
 #
-# GH_PAT - github Personal Access Token for accessing PRIVATE_CONFIG_REPO
+# GH_PAT - github Personal Access Token for accessing private config repo
 #
-# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if app servers are hosted on private network.
+# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if API nodes live on private network.
+# SSH_JUMPHOST_USER - username  to use to connect to the ssh jump/proxy.
 #
-# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/deploy_id_rsa_enc (see miloserdow/capistrano-deploy)
-# this SSH key is used for accessing jump host, UI nodes, and private github repo.
+# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/
+# this SSH key is used for accessing jump host, API nodes, and private github repo.
 
 name: Capistrano Deployment
 # Controls when the action will run.
 on:
-  # Trigger deployment to staging after unit test action completes
-  workflow_run:
-    workflows: ["Ruby Unit Tests"]
-    types:
-      - completed
-    branches: [master, develop]
+  push:
+    branches:
+      - stage
+      - test
   # Allows running this workflow manually from the Actions tab
   workflow_dispatch:
-    branches: [master, develop]
     inputs:
       BRANCH:
-        description: 'Branch/tag to deploy'
-        default: develop
+        description: "Branch/tag to deploy"
+        options:
+          - stage
+          - test
+          - master
+        default: stage
         required: true
       environment:
-        description: 'target environment to deploy to'
+        description: "target environment to deploy to"
         type: choice
         options:
           - staging
-          - production
-        default: staging
-
+          - agroportal
+          - test
+        default: stage
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    # run deployment only if "Ruby Unit Tests" workflow completes sucessefully or when manually triggered
-    if: ${{ (github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch') }}
     env:
-      BUNDLE_WITHOUT: default #install gems required primarily for the deployment in order to speed this workflow
+      BUNDLE_WITHOUT: default #install gems required primarely for deployment in order to speed up workflow
       PRIVATE_CONFIG_REPO: ${{ format('[email protected]:{0}.git', secrets.CONFIG_REPO) }}
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-    - name: set branch/tag and environment to deploy from inputs
-      run: |
-        # workflow_dispatch default input doesn't get set on push so we need to set defaults
-        # via shell parameter expansion
-        # https://dev.to/mrmike/github-action-handling-input-default-value-5f2g
-        USER_INPUT_BRANCH=${{ inputs.branch }}
-        echo "BRANCH=${USER_INPUT_BRANCH:-develop}" >> $GITHUB_ENV
-        USER_INPUT_ENVIRONMENT=${{ inputs.environment }}
-        echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 2.7.6 # Not needed with a .ruby-version file
-        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: get-deployment-config
-      uses: actions/checkout@v3
-      with:
-        repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings
-        token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT
-        path:  deploy_config
-    - name: copy-deployment-config
-      run:  cp -r deploy_config/ontologies_api/* .
-    # add ssh hostkey so that capistrano doesn't complain
-    - name: Add jumphost's hostkey to Known Hosts
-      run: |
-        mkdir -p ~/.ssh
-        ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts
-      shell: bash
-    - uses: miloserdow/capistrano-deploy@master
-      with:
-        target: ${{ env.TARGET }} # which environment to deploy
-        deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project
+      - name: set branch/tag and environment to deploy from inputs
+        run: |
+          # workflow_dispatch default input doesn't get set on push so we need to set defaults
+          # via shell parameter expansion
+          # https://dev.to/mrmike/github-action-handling-input-default-value-5f2g
+          USER_INPUT_BRANCH=${{ inputs.branch }}
+          echo "BRANCH=${USER_INPUT_BRANCH:github.head_ref:-master}" >> $GITHUB_ENV
+
+          USER_INPUT_ENVIRONMENT=${{ inputs.environment }}
+          echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV
+
+          CONFIG_REPO=${{ secrets.CONFIG_REPO }}
+          GH_PAT=${{ secrets.GH_PAT }}
+          echo "PRIVATE_CONFIG_REPO=https://${GH_PAT}@github.com/${CONFIG_REPO}" >> $GITHUB_ENV
+
+          echo "SSH_JUMPHOST=${{ secrets.SSH_JUMPHOST }}" >> $GITHUB_ENV
+          echo "SSH_JUMPHOST_USER=${{ secrets.SSH_JUMPHOST_USER }}" >> $GITHUB_ENV
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7.6 # Not needed with a .ruby-version file
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: get-deployment-config
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings
+          token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT
+          path: deploy_config
+      - name: copy-deployment-config
+        run: cp -r deploy_config/ontologies_api/${{ inputs.environment }}/* .
+        # add ssh hostkey so that capistrano doesn't complain
+      - name: Add jumphost's hostkey to Known Hosts
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_JUMPHOST }}"
+          ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts
+        shell: bash
+      - uses: miloserdow/capistrano-deploy@master
+        with:
+          target: ${{ env.TARGET }} # which environment to deploy
+          deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project

.gitignore

@@ -36,7 +36,6 @@ config/environments/*
 !config/environments/config.rb.sample
 
 #ignore capistrano deployment
-config/deploy/*
 config/*.p12
 
 # Ignore generated test data

Gemfile.lock

@@ -121,7 +121,7 @@ GEM
       i18n
       rake (>= 10.0.0)
       sshkit (>= 1.9.0)
-    capistrano-bundler (2.1.0)
+    capistrano-bundler (2.1.1)
       capistrano (~> 3.1)
     capistrano-locally (0.3.0)
       capistrano (~> 3.0)
@@ -178,7 +178,7 @@ GEM
     google-analytics-data (0.6.0)
       google-analytics-data-v1beta (>= 0.11, < 2.a)
       google-cloud-core (~> 1.6)
-    google-analytics-data-v1beta (0.12.0)
+    google-analytics-data-v1beta (0.13.0)
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
     google-apis-analytics_v3 (0.16.0)
@@ -191,7 +191,7 @@ GEM
       mutex_m
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
-    google-cloud-core (1.7.0)
+    google-cloud-core (1.7.1)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
     google-cloud-env (2.1.1)

config/deploy.rb

@@ -1,17 +1,14 @@
-# config valid only for Capistrano 3
-
-APP_PATH = '/srv/ontoportal'
-
-set :application, 'ontologies_api'
-set :repo_url, "https://github.com/ncbo/#{fetch(:application)}.git"
+set :author, "ontoportal-lirmm"
+set :application, "ontologies_api"
+set :repo_url, "https://github.com/#{fetch(:author)}/#{fetch(:application)}.git"
 
 set :deploy_via, :remote_cache
 
 # Default branch is :master
 # ask :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp }
 
 # Default deploy_to directory is /var/www/my_app
-set :deploy_to, "#{APP_PATH}/#{fetch(:application)}"
+set :deploy_to, "/srv/ontoportal/#{fetch(:application)}"
 
 # Default value for :scm is :git
 # set :scm, :git
@@ -20,7 +17,7 @@
 # set :format, :pretty
 
 # Default value for :log_level is :debug
-# set :log_level, :debug
+set :log_level, :error
 
 # Default value for :pty is false
 # set :pty, true
@@ -32,21 +29,40 @@
 # set :linked_dirs, %w{log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system}
 set :linked_dirs, %w{log vendor/bundle tmp/pids tmp/sockets public/system}
 
-# rbenv
-# set :rbenv_type, :system #or :user
-# set :rbenv_ruby, '2.2.5'
-# set :rbenv_roles, :all # default value
-
-# do not use sudo
-set :use_sudo, false
-# required for restarting unicorn with sudo
-set :pty, true
 # Default value for default_env is {}
-set :default_env, {
-}
+# set :default_env, { path: "/opt/ruby/bin:$PATH" }
 
 # Default value for keep_releases is 5
 set :keep_releases, 5
+set :config_folder_path, "#{fetch(:application)}/#{fetch(:stage)}"
+
+# If you want to restart using `touch tmp/restart.txt`, add this to your config/deploy.rb:
+
+SSH_JUMPHOST = ENV.include?('SSH_JUMPHOST') ? ENV['SSH_JUMPHOST'] : 'jumpbox.hostname.com'
+SSH_JUMPHOST_USER = ENV.include?('SSH_JUMPHOST_USER') ? ENV['SSH_JUMPHOST_USER'] : 'username'
+
+JUMPBOX_PROXY = "#{SSH_JUMPHOST_USER}@#{SSH_JUMPHOST}"
+set :ssh_options, {
+  user: 'ontoportal',
+  forward_agent: 'true',
+  keys: %w(config/deploy_id_rsa),
+  auth_methods: %w(publickey),
+  # use ssh proxy if API servers are on a private network
+  proxy: Net::SSH::Proxy::Command.new("ssh #{JUMPBOX_PROXY} -W %h:%p")
+}
+
+# private git repo for configuraiton
+PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'https://[email protected]/your_organization/ontoportal-configs.git'
+desc "Check if agent forwarding is working"
+task :forwarding do
+  on roles(:all) do |h|
+    if test("env | grep SSH_AUTH_SOCK")
+      info "Agent forwarding is up to #{h}"
+    else
+      error "Agent forwarding is NOT up to #{h}"
+    end
+  end
+end
 
 # inspired by http://nathaniel.talbott.ws/blog/2013/03/14/post-deploy-smoke-tests/
 desc 'Run smoke test'
@@ -74,18 +90,17 @@
   end
 end
 
-
 namespace :deploy do
 
   desc 'Incorporate the private repository content'
   # Get cofiguration from repo if PRIVATE_CONFIG_REPO env var is set
   # or get config from local directory if LOCAL_CONFIG_PATH env var is set
   task :get_config do
     if defined?(PRIVATE_CONFIG_REPO)
-      TMP_CONFIG_PATH = "/tmp/#{SecureRandom.hex(15)}"
+      TMP_CONFIG_PATH = "/tmp/#{SecureRandom.hex(15)}".freeze
       on roles(:app) do
         execute "git clone -q #{PRIVATE_CONFIG_REPO} #{TMP_CONFIG_PATH}"
-        execute "rsync -av #{TMP_CONFIG_PATH}/#{fetch(:application)}/ #{release_path}/"
+        execute "rsync -av #{TMP_CONFIG_PATH}/#{fetch(:config_folder_path)}/ #{release_path}/"
         execute "rm -rf #{TMP_CONFIG_PATH}"
       end
     elsif defined?(LOCAL_CONFIG_PATH)
@@ -98,16 +113,15 @@
   desc 'Restart application'
   task :restart do
     on roles(:app), in: :sequence, wait: 5 do
-    # Your restart mechanism here, for example:
-    # execute :touch, release_path.join('tmp/restart.txt')
-    execute 'sudo systemctl restart unicorn'
-    execute 'sleep 5'
+      # Your restart mechanism here, for example:
+      # execute :touch, release_path.join('tmp/restart.txt')
+      execute 'sudo systemctl restart unicorn'
+      execute 'sleep 5'
     end
   end
 
-  after :publishing, :get_config
-  after :get_config, :restart
-  # after :deploy, :smoke_test
+  after :updating, :get_config
+  after :publishing, :restart
 
   after :restart, :clear_cache do
     on roles(:web), in: :groups, limit: 3, wait: 10 do

config/deploy/agroportal.rb

@@ -0,0 +1,17 @@
+# Simple Role Syntax
+# ==================
+# Supports bulk-adding hosts to roles, the primary
+# server in each group is considered to be the first
+# unless any hosts have the primary property set.
+# Don't declare `role :all`, it's a meta role
+role :app, %w[agroportal.lirmm.fr]
+role :db, %w[agroportal.lirmm.fr] # sufficient to run db:migrate only on one system
+set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master'
+# Extended Server Syntax
+# ======================
+# This can be used to drop a more detailed server
+# definition into the server list. The second argument
+# something that quacks like a hash can be used to set
+# extended properties on the server.
+# server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value
+set :log_level, :error