refactor: datacollector (#2240)

- separate business logic in Module Datacollector:

  -- collector.rb: file/folder crawler for a given device
  --  correspondence.rb: set the sender and recipient
     encompass AR Model logic to create the attachments into the proper
     containers
  -- configuration.rb: validate the datacollector device config
     also instantiate the SFTP client if applicable
  -- collector_file: file/folder class for basic file operations
      independent of whether the file is on local fs or over sftp

-   AdminApi device connection testing:
    -- now rely on Datacollector::Configuration

- mailcollector and configuration

-  DRY dev seeds for devices to use spec/factories

* feat(UI): add a `disabled` option in the collector method to allow keeping the config while disabling collection for the device

.env.test → .env.test.example

@@ -1,8 +1,8 @@
-SFTP_HOST=sftp
-SFTP_PORT=22
-SFTP_UPLOAD_FOLDER=uploads
-SFTP_USER=sftp_test
-SFTP_PASSWORD=sftp_test
+# SFTP_HOST=sftp
+# SFTP_PORT=22
+# SFTP_UPLOAD_FOLDER=uploads
+# SFTP_USER=sftp_test
+# SFTP_PASSWORD=sftp_test
 
 MESSAGE_ENABLE=true
 MESSAGE_AUTO_INTERNAL=1000
@@ -29,3 +29,9 @@ DATA_CITE_DEVICE_CREATOR=chemotion.net
 
 # uncomment for headfull testing
 # USE_HEAD=:D
+
+## DEVICE DATACOLLECTOR FACTORY CONFIG (SFTP and local dir)
+# DATACOLLECTOR_FACTORY_SFTP_USER=testuser
+# DATACOLLECTOR_FACTORY_SFTP_KEY=id_test
+# DATACOLLECTOR_FACTORY_SFTP_HOST=127.0.0.1
+# DATACOLLECTOR_FACTORY_DIR=

.github/workflows/ci.yml

@@ -72,22 +72,17 @@ jobs:
           mkdir /home/testuser/.ssh
           chown testuser:testuser /home/testuser/.ssh
           chmod 700 /home/testuser/.ssh
-
-      - name: configure ssh for datacollectors testing 
-        run: |
-          service ssh restart
           mkdir -p $HOME/.ssh
           chmod 700 $HOME/.ssh
+          service ssh restart
           ssh-keygen -t ed25519 -f $HOME/.ssh/id_test -N ""
-          # echo "$(<$HOME/.ssh/id_test.pub)" >> $HOME/.ssh/authorized_keys
-          echo "$(<$HOME/.ssh/id_test.pub)" | sudo tee -a /home/testuser/.ssh/authorized_keys
-          eval `ssh-agent`
-          ssh-add $HOME/.ssh/id_test
+          cat "${HOME}/.ssh/id_test.pub" | tee -a /home/testuser/.ssh/authorized_keys
           ssh-keyscan -H 127.0.0.1  >> $HOME/.ssh/known_hosts
 
       - name: configure repository
         run: |
           mv public/welcome-message-sample.md public/welcome-message.md
+          cp .env.test.example .env.test
           cd config
           cp database.yml.gitlab database.yml
           cp -f datacollectors.yml.example datacollectors.yml
@@ -126,8 +121,8 @@ jobs:
 
       - name: rspec unit
         run: |
+          service ssh restart
           eval `ssh-agent`
-          ssh-add $HOME/.ssh/id_test
           RAILS_ENV=test bundle exec rspec --exclude-pattern spec/{features}/**/*_spec.rb spec
 
       - name: coverage rspec unit
@@ -139,10 +134,3 @@ jobs:
           artifact-name: code-coverage-report
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-#      - name: precompile
-#        run: RAILS_ENV=test bundle exec rake webpacker:compile
-
-#      - name: rspec acceptance
-#        continue-on-error: true    # don't fail job because this step; TODO: fix flaky acceptance tests...
-#        run: RAILS_ENV=test bundle exec rspec spec/features
-

.gitignore

@@ -24,11 +24,14 @@
 /tmp/novnc_devices/*
 !/tmp/novnc_devices/.keep
 
+/.yardoc/
+
 .ruby-gemset
 .ruby-version
 .coveralls.yml
 
 .env
+.env.test
 
 /config/matrices.json
 /config/klasses.json
@@ -220,5 +223,5 @@ data/klasses.json
 public/pdf.worker.min.js
 
 # Sentry Config File
-.env.sentry-build-plugin
+.env.sentry-build-plugin*
 

app/api/chemotion/admin_api.rb

@@ -8,7 +8,6 @@
 module Chemotion
   # Publish-Subscription MessageAPI
   class AdminAPI < Grape::API
-    helpers AdminHelpers
     resource :admin do
       before do
         error!(401) unless current_user.is_a?(Admin)

app/api/chemotion/admin_device_api.rb

@@ -2,7 +2,6 @@
 
 module Chemotion
   class AdminDeviceAPI < Grape::API
-    helpers AdminHelpers
     resource :admin_devices do
       before do
         error!(401) unless current_user.is_a?(Admin)
@@ -127,7 +126,7 @@ class AdminDeviceAPI < Grape::API
         end
       end
 
-      # test datacollector sftp
+      # test datacollector sftp connection
       params do
         requires :id, type: Integer
         optional :datacollector_method, type: String
@@ -138,21 +137,9 @@ class AdminDeviceAPI < Grape::API
       end
       route_param :test_sftp do
         post do
-          case params[:datacollector_authentication]
-          when 'password'
-            credentials = Rails.configuration.datacollectors.sftpusers.find do |e|
-              e[:user] == params[:datacollector_user]
-            end
-            raise 'No match user credentials!' unless credentials
-
-            connect_sftp_with_password(
-              host: params[:datacollector_host],
-              user: credentials[:user],
-              password: credentials[:password],
-            )
-          when 'keyfile'
-            connect_sftp_with_key(params)
-          end
+          # make options hashie compatible
+          options = Hashie::Mash.new declared(params, include_missing: false).merge(info: params[:id])
+          Datacollector::Configuration.new!(options)
 
           { status: 'success', message: 'Test connection successfully.' }
         rescue StandardError => e