diff --git a/.env.test b/.env.test.example similarity index 71% rename from .env.test rename to .env.test.example index 136fc0fe9a..331dd5f90f 100644 --- a/.env.test +++ b/.env.test.example @@ -1,8 +1,8 @@ -SFTP_HOST=sftp -SFTP_PORT=22 -SFTP_UPLOAD_FOLDER=uploads -SFTP_USER=sftp_test -SFTP_PASSWORD=sftp_test +# SFTP_HOST=sftp +# SFTP_PORT=22 +# SFTP_UPLOAD_FOLDER=uploads +# SFTP_USER=sftp_test +# SFTP_PASSWORD=sftp_test MESSAGE_ENABLE=true MESSAGE_AUTO_INTERNAL=1000 @@ -29,3 +29,9 @@ DATA_CITE_DEVICE_CREATOR=chemotion.net # uncomment for headfull testing # USE_HEAD=:D + +## DEVICE DATACOLLECTOR FACTORY CONFIG (SFTP and local dir) +# DATACOLLECTOR_FACTORY_SFTP_USER=testuser +# DATACOLLECTOR_FACTORY_SFTP_KEY=id_test +# DATACOLLECTOR_FACTORY_SFTP_HOST=127.0.0.1 +# DATACOLLECTOR_FACTORY_DIR= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c70b097ed6..72c68bc77d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,22 +72,17 @@ jobs: mkdir /home/testuser/.ssh chown testuser:testuser /home/testuser/.ssh chmod 700 /home/testuser/.ssh - - - name: configure ssh for datacollectors testing - run: | - service ssh restart mkdir -p $HOME/.ssh chmod 700 $HOME/.ssh + service ssh restart ssh-keygen -t ed25519 -f $HOME/.ssh/id_test -N "" - # echo "$(<$HOME/.ssh/id_test.pub)" >> $HOME/.ssh/authorized_keys - echo "$(<$HOME/.ssh/id_test.pub)" | sudo tee -a /home/testuser/.ssh/authorized_keys - eval `ssh-agent` - ssh-add $HOME/.ssh/id_test + cat "${HOME}/.ssh/id_test.pub" | tee -a /home/testuser/.ssh/authorized_keys ssh-keyscan -H 127.0.0.1 >> $HOME/.ssh/known_hosts - name: configure repository run: | mv public/welcome-message-sample.md public/welcome-message.md + cp .env.test.example .env.test cd config cp database.yml.gitlab database.yml cp -f datacollectors.yml.example datacollectors.yml @@ -126,8 +121,8 @@ jobs: - name: rspec unit run: | + service ssh restart eval `ssh-agent` - ssh-add $HOME/.ssh/id_test RAILS_ENV=test bundle exec rspec --exclude-pattern spec/{features}/**/*_spec.rb spec - name: coverage rspec unit @@ -139,10 +134,3 @@ jobs: artifact-name: code-coverage-report github-token: ${{ secrets.GITHUB_TOKEN }} -# - name: precompile -# run: RAILS_ENV=test bundle exec rake webpacker:compile - -# - name: rspec acceptance -# continue-on-error: true # don't fail job because this step; TODO: fix flaky acceptance tests... -# run: RAILS_ENV=test bundle exec rspec spec/features - diff --git a/.gitignore b/.gitignore index df35592aa2..1be9aef048 100644 --- a/.gitignore +++ b/.gitignore @@ -24,11 +24,14 @@ /tmp/novnc_devices/* !/tmp/novnc_devices/.keep +/.yardoc/ + .ruby-gemset .ruby-version .coveralls.yml .env +.env.test /config/matrices.json /config/klasses.json @@ -220,5 +223,5 @@ data/klasses.json public/pdf.worker.min.js # Sentry Config File -.env.sentry-build-plugin +.env.sentry-build-plugin* diff --git a/app/api/chemotion/admin_api.rb b/app/api/chemotion/admin_api.rb index 79ae566fd4..06c2822f04 100644 --- a/app/api/chemotion/admin_api.rb +++ b/app/api/chemotion/admin_api.rb @@ -8,7 +8,6 @@ module Chemotion # Publish-Subscription MessageAPI class AdminAPI < Grape::API - helpers AdminHelpers resource :admin do before do error!(401) unless current_user.is_a?(Admin) diff --git a/app/api/chemotion/admin_device_api.rb b/app/api/chemotion/admin_device_api.rb index c3c3646546..aaeb8b46bf 100644 --- a/app/api/chemotion/admin_device_api.rb +++ b/app/api/chemotion/admin_device_api.rb @@ -2,7 +2,6 @@ module Chemotion class AdminDeviceAPI < Grape::API - helpers AdminHelpers resource :admin_devices do before do error!(401) unless current_user.is_a?(Admin) @@ -127,7 +126,7 @@ class AdminDeviceAPI < Grape::API end end - # test datacollector sftp + # test datacollector sftp connection params do requires :id, type: Integer optional :datacollector_method, type: String @@ -138,21 +137,9 @@ class AdminDeviceAPI < Grape::API end route_param :test_sftp do post do - case params[:datacollector_authentication] - when 'password' - credentials = Rails.configuration.datacollectors.sftpusers.find do |e| - e[:user] == params[:datacollector_user] - end - raise 'No match user credentials!' unless credentials - - connect_sftp_with_password( - host: params[:datacollector_host], - user: credentials[:user], - password: credentials[:password], - ) - when 'keyfile' - connect_sftp_with_key(params) - end + # make options hashie compatible + options = Hashie::Mash.new declared(params, include_missing: false).merge(info: params[:id]) + Datacollector::Configuration.new!(options) { status: 'success', message: 'Test connection successfully.' } rescue StandardError => e diff --git a/app/api/helpers/admin_helpers.rb b/app/api/helpers/admin_helpers.rb deleted file mode 100644 index 547d69ce23..0000000000 --- a/app/api/helpers/admin_helpers.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -# A helper for admin_api -module AdminHelpers - extend Grape::API::Helpers - - def key_path(key_name) - key_dir = Rails.configuration.datacollectors.keydir - kp = if key_dir.start_with?('/') - Pathname.new(key_dir).join(key_name) - else - Rails.root.join(key_dir, key_name) - end - error!('No key file found', 500) unless kp.file? && kp.exist? - kp - end - - def connect_sftp_with_password(prms) - options = { - password: prms[:password], - auth_methods: ['password'], - number_of_password_prompts: 0, - } - sftp_start_with_options(prms, options) - end - - def connect_sftp_with_key(prms) - options = { - key_data: [], - keys: key_path(prms[:datacollector_key_name]), - keys_only: true, - } - sftp_start_with_options(prms, options) - end - - def sftp_start_with_options(prms, options) - uri = URI.parse("ssh://#{prms[:datacollector_host]}") - options[:port] = uri.port if uri.port - options[:timeout] = 5 - options[:non_interactive] = true - sftp = Net::SFTP.start( - uri.host, - prms[:datacollector_user], - **options, - ) - raise 'Connection can not be initialized!' unless sftp.open? - ensure - sftp.nil? || sftp.close_channel - end -end diff --git a/app/clients/sftp_client.rb b/app/clients/sftp_client.rb index 5b5ba27a73..810e82eff2 100644 --- a/app/clients/sftp_client.rb +++ b/app/clients/sftp_client.rb @@ -1,27 +1,57 @@ +# frozen_string_literal: true + class SFTPClient + ALLOWED_OPTIONS = %i[ + port password keys keys_only auth_methods timeout verbose key_data number_of_password_prompts + ].freeze + def self.with_default_settings - @with_default_settings ||= self.new( - { - host: ENV['SFTP_HOST'], - port: ENV['SFTP_PORT'], - username: ENV['SFTP_USER'], - password: (ENV['SFTP_PASSWORD'] if ENV['SFTP_PASSWORD']), - keys: (ENV['SFTP_KEYS'] if ENV['SFTP_KEYS']) - } + @with_default_settings ||= new( + host: ENV.fetch('SFTP_HOST', nil), + user: ENV.fetch('SFTP_USER', nil), + + port: ENV.fetch('SFTP_PORT', nil), + password: ENV.fetch('SFTP_PASSWORD', nil), + keys: ENV.fetch('SFTP_KEYS', nil), + auth_methods: ENV.fetch('SFTP_AUTH_METHODS', nil), ) end - def initialize(sftp_config) - @host = sftp_config.fetch(:host, nil) - @port = sftp_config.fetch(:port, nil) - @username = sftp_config.fetch(:username, nil) - @password = sftp_config.fetch(:password, nil) + attr_reader :host, :user, :session_options - # This specifies the list of private key files to use instead of the - # defaults ($HOME/.ssh/id_dsa, $HOME/.ssh2/id_dsa, $HOME/.ssh/id_rsa, and $HOME/.ssh2/id_rsa). - # The value of this option should be an array of strings. - # See http://net-ssh.github.io/ssh/v1/chapter-2.html - @keys = sftp_config.fetch(:keys, []) + # Initialize the SFTP parameters with the given host, username and options + # to be passed to the Net::SFTP.start method. + # + # @param host [String] the host to connect to + # @param user [String] the username to use for the connection + # @param options [Hash] the options to pass to the Net::SFTP.start method + # @options options [String] :port the port to connect to + # @options options [String] :password the password to use for the connection + # @options options [Array] :keys the key files to use for the connection + # @options options [Boolean] :keys_only whether to use only the keys for the connection + # @options options [Array] :auth_methods the authentication methods to use + # @options options [Integer] :timeout the timeout to use for the connection + # @options options [Boolean] :verbose whether to be verbose + # @options options [Array] :key_data the key data to use for the connection + # @example new('example.com', 'user', { port: '2222', password: 'password' }) + # @example new('sfpt://john@foo.bar:1234/unrelevant') + # @note see ALLOWED_OPTIONS for the allowed options + # @note optional user and port params have precedence over the parsed ones from the host + def initialize(host = nil, user = nil, **options) + extract_host_and_user(host, user, **options) + raise ArgumentError, 'No host or user given' unless @host && @user + + %i[auth_methods keys].each do |key| + options[key] = options[key].split(',') if options[key].is_a?(String) + end + default_options(options) + @session_options = @session_options.merge(options.slice(*ALLOWED_OPTIONS)).compact + end + + # Test the connection to the SFTP server + # @return [Boolean] whether the connection was successful + def open? + with_session(&:open?) end def upload_file!(local_path, remote_path) @@ -47,6 +77,12 @@ def download_file!(remote_path, local_path) end end + def download_directory!(remote_path, local_path) + with_session(remote_path, local_path) do |sftp| + sftp.download!(remote_path, local_path, requests: 5, recursive: true) + end + end + def move_file!(remote_src_path, remote_target_path) with_session(remote_src_path, remote_target_path) do |sftp| sftp.rename(remote_src_path, remote_target_path).wait @@ -56,7 +92,13 @@ def move_file!(remote_src_path, remote_target_path) def remove_file!(remote_path) with_session(remote_path) do |sftp| - sftp.remove(remote_path).wait + sftp.remove!(remote_path) + end + end + + def remove_dir!(remote_path) + with_session(remote_path) do |sftp| + sftp.session.exec!("rm -rf #{remote_path}") end end @@ -66,40 +108,80 @@ def read_file(remote_path) end end - def file_exists?(remote_path) + def exist?(remote_path) + with_session do |sftp| + sftp.stat!(remote_path) && true + rescue Net::SFTP::StatusException => e + return false if e.code == 2 # No such file + + raise e + end + end + + def file?(remote_path) + parent = File.dirname(remote_path) + name = File.basename(remote_path) + result = nil + with_session do |sftp| + result = sftp.dir.glob(parent, name).find(&:file?).present? + end + result + end + + def directory?(remote_path) + parent = File.dirname(remote_path) + name = File.basename(remote_path) + result = nil + with_session(parent) do |sftp| + result = sftp.dir.glob(parent, name).find(&:directory?).present? + end + result + end + + def entries(remote_path) with_session(remote_path) do |sftp| - begin - sftp.stat!(remote_path) - rescue Net::SFTP::StatusException => e - if e.code == 2 # No such file - return false - else - raise e - end - end - true + sftp.dir.entries(remote_path) + end + end + + def glob(remote_path, pattern, flags = 0) + with_session do |sftp| + sftp.dir.glob(remote_path, pattern, flags) end end private - def with_session(*args_of_caller) - begin - if @keys.blank? - Net::SFTP.start(@host, @username, port: @port, password: @password) do |sftp| - data = yield(sftp) - return data if data - end - else - Net::SFTP.start(@host, @username, port: @port, keys: @keys) do |sftp| - data = yield(sftp) - return data if data - end - end - rescue Exception => e - # for usage of caller, see Kernel#caller - raise SFTPClientError.new(e, caller[1], args_of_caller) - end - end + # Extract Host, User and Port from the given URI + def extract_host_and_user(host, user, **options) + # extract host information from the given host, remove protocol prefix if present + host = (host.presence || options.delete(:host)).sub(%r{^[a-z]+://}, '') + uri = URI.parse("ssh://#{host}") + @host = uri.host + @user = user || uri.user || options.delete(:user) + @root_path = options.delete(:root_path) || (uri.path != '/' && uri.path.presence) + @session_options = { port: uri.port } + end + + # Set some default options + def default_options(options) + @session_options = @session_options.merge( + timeout: 5, verbose: :warn, keys_only: true, auth_methods: [], number_of_password_prompts: 0, + ) + @session_options[:auth_methods] = %w[publickey] if options[:keys].present? || options[:key_data].present? + @session_options[:auth_methods] += %w[password] if options[:password].present? + @session_options[:auth_methods] = %w[publickey] if options[:keys_only].present? + end + # rubocop:disable Lint/RescueException + def with_session(*args_of_caller) + Net::SFTP.start(@host, @user, @session_options) do |sftp| + data = yield(sftp) + return data unless data.nil? + end + rescue Exception => e + # for usage of caller, see Kernel#caller + raise SFTPClientError.new(e, caller(2..2).first, args_of_caller) + end + # rubocop:enable Lint/RescueException end diff --git a/app/errors/errors.rb b/app/errors/errors.rb index 5c02cdcb24..80c9aef5ad 100644 --- a/app/errors/errors.rb +++ b/app/errors/errors.rb @@ -14,4 +14,6 @@ def initialize(message = 'Forbidden') super(message) end end + + class DatacollectorError < ApplicationError; end end diff --git a/app/jobs/collect_data_from_local_job.rb b/app/jobs/collect_data_from_local_job.rb index f47bdcb2bd..a8b0f5f532 100644 --- a/app/jobs/collect_data_from_local_job.rb +++ b/app/jobs/collect_data_from_local_job.rb @@ -2,7 +2,7 @@ class CollectDataFromLocalJob < ApplicationJob queue_as :collect_data def perform - collector = Foldercollector.new - collector.execute(false) + devices = Device.where(datacollector_method: 'folderwatcherlocal') + Datacollector::Collector.bulk_execute(devices) end end diff --git a/app/jobs/collect_data_from_mail_job.rb b/app/jobs/collect_data_from_mail_job.rb index a3b59c0ef8..bb77f51e7f 100644 --- a/app/jobs/collect_data_from_mail_job.rb +++ b/app/jobs/collect_data_from_mail_job.rb @@ -1,8 +1,8 @@ class CollectDataFromMailJob < ApplicationJob queue_as :collect_data - + def perform - collector = Mailcollector.new + collector = Datacollector::Mailcollector.new collector.execute end end diff --git a/app/jobs/collect_data_from_sftp_job.rb b/app/jobs/collect_data_from_sftp_job.rb index 94f7eb7ff9..2b4f535486 100644 --- a/app/jobs/collect_data_from_sftp_job.rb +++ b/app/jobs/collect_data_from_sftp_job.rb @@ -2,7 +2,7 @@ class CollectDataFromSftpJob < ApplicationJob queue_as :collect_data def perform - collector = Foldercollector.new - collector.execute(true) + devices = Device.where(datacollector_method: 'folderwatchersftp') + Datacollector::Collector.bulk_execute(devices) end end diff --git a/app/jobs/collect_file_from_local_job.rb b/app/jobs/collect_file_from_local_job.rb index 2d8a09b46c..b0328e8727 100644 --- a/app/jobs/collect_file_from_local_job.rb +++ b/app/jobs/collect_file_from_local_job.rb @@ -2,7 +2,7 @@ class CollectFileFromLocalJob < ApplicationJob queue_as :collect_data def perform - collector = Filecollector.new - collector.execute(false) + devices = Device.where(datacollector_method: 'filewatcherlocal') + Datacollector::Collector.bulk_execute(devices) end end diff --git a/app/jobs/collect_file_from_sftp_job.rb b/app/jobs/collect_file_from_sftp_job.rb index 93d068f0ec..eae271c933 100644 --- a/app/jobs/collect_file_from_sftp_job.rb +++ b/app/jobs/collect_file_from_sftp_job.rb @@ -1,8 +1,8 @@ class CollectFileFromSftpJob < ApplicationJob queue_as :collect_data - + def perform - collector = Filecollector.new - collector.execute(true) + devices = Device.where(datacollector_method: 'filewatchersftp') + Datacollector::Collector.bulk_execute(devices) end end diff --git a/app/models/collector_error.rb b/app/models/collector_error.rb index dd14839e45..9132b0447b 100644 --- a/app/models/collector_error.rb +++ b/app/models/collector_error.rb @@ -8,5 +8,31 @@ # updated_at :datetime not null # +# Register if a file/folder could not be fully processed by the Collector +# so that the same target is not processed again in the future. +# (For example when the target can not be deleted) class CollectorError < ApplicationRecord + # Find a record by path + # @param file_path [String] The file path + # @param mtime [Time] Timestamp + # @return [CollectorError] + def self.find_by_path(path, mtime) + error_code = hash(path, mtime) + find_by(error_code: error_code) + end + + # Find or create a record by path + # (see #self.find_or_create_by_path) + def self.find_or_create_by_path(path, mtime = nil) + error_code = hash(path, mtime) + find_or_create_by(error_code: error_code) + end + + # Generate a hash from file_path and modified_at timestamp that can be used for error tracking + # @param (see #find_by_path) + # @return [String] the digest + def self.hash(path, mtime) + # Digest::SHA256.hexdigest(path + date.to_s) + Digest::SHA256.hexdigest(path + mtime.to_i.to_s) + end end diff --git a/app/models/device.rb b/app/models/device.rb index 7bd480507a..0b9948f756 100644 --- a/app/models/device.rb +++ b/app/models/device.rb @@ -77,6 +77,7 @@ class Device < ApplicationRecord scope :by_user_ids, ->(ids) { joins(:users_devices).merge(UsersDevice.by_user_ids(ids)) } scope :by_name, ->(query) { where('LOWER(name) ILIKE ?', "%#{sanitize_sql_like(query.downcase)}%") } + scope :by_email, ->(query) { where('LOWER(email) ILIKE ?', sanitize_sql_like(query.downcase.strip)) } def unique_name_abbreviation devices = Device.unscoped.where('LOWER(name_abbreviation) = ?', diff --git a/app/models/user.rb b/app/models/user.rb index 7e7c77e73f..b79ce94bec 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -149,6 +149,13 @@ class User < ApplicationRecord end } + # Find user by an email address in a case insensitive way + # @param [String] email + # @return [ActiveRecord::Relation] + scope :by_email, lambda { |query| + where('LOWER(email) = ?', sanitize_sql_like(query.downcase.strip)) + } + # try to find a user by exact match of name_abbreviation # fall back to insensitive match result unless multiple users are found. def self.try_find_by_name_abbreviation(name_abbreviation) diff --git a/app/packs/src/apps/admin/devices/DeviceDataCollectorTab.js b/app/packs/src/apps/admin/devices/DeviceDataCollectorTab.js index 5f37a08f7e..fcc25c61dc 100644 --- a/app/packs/src/apps/admin/devices/DeviceDataCollectorTab.js +++ b/app/packs/src/apps/admin/devices/DeviceDataCollectorTab.js @@ -25,7 +25,8 @@ const DeviceDataCollectorTab = () => { { value: 'filewatchersftp', label: 'filewatchersftp' }, { value: 'filewatcherlocal', label: 'filewatcherlocal' }, { value: 'folderwatchersftp', label: 'folderwatchersftp' }, - { value: 'folderwatcherlocal', label: 'folderwatcherlocal' } + { value: 'folderwatcherlocal', label: 'folderwatcherlocal' }, + { value: 'disabled', label: 'disabled' } ]; const authenticationOptions = [ diff --git a/config/datacollectors.yml.example b/config/datacollectors.yml.example index f17eeefafc..ac1be186e9 100644 --- a/config/datacollectors.yml.example +++ b/config/datacollectors.yml.example @@ -1,18 +1,20 @@ shared: # List of enabled data collection services :services: - # - :name: 'mailcollector' + - :name: 'mailcollector' # :cron: '5,20,35,50 * * * *' # every 15 minutes starting a xx:05 - # - :name: 'folderwatchersftp' - # :every: 5 # minutes - # :watcher_sleep: 30 # seconds - # - :name: 'folderwatcherlocal' - # :every: 5 # minutes - # :watcher_sleep: 30 # seconds - # - :name: 'filewatchersftp' - # :every: 2 # minutes - # - :name: 'filewatcherlocal' - # :every: 2 # minutes + - :name: 'folderwatchersftp' + # :every: 5 # every 5 minutes + # :watcher_sleep: 30 # seconds: skip if file/folder mtime newer than x seconds + - :name: 'folderwatcherlocal' + # :every: 5 + # :watcher_sleep: 30 + - :name: 'filewatchersftp' + # :every: 2 + # :watcher_sleep: 30 + - :name: 'filewatcherlocal' + # :every: 2 + # :watcher_sleep: 30 # Email service configuration :mailcollector: @@ -57,6 +59,25 @@ development: &development - :path: '<%= Rails.root.join(*%w[tmp datacollector]).to_s %>' test: - <<: *development + :services: + - :name: 'folderwatcherlocal' + :watcher_sleep: 0 + - :name: 'filewatcherlocal' + :watcher_sleep: 0 + - :name: 'folderwatchersftp' + :watcher_sleep: 0 + - :name: 'filewatchersftp' + :watcher_sleep: 0 + :keydir: '<%= "#{Dir.home}/.ssh/" %>' + :localcollectors: + - :path: '<%= Dir.mktmpdir(%w[chemotion_collector_test-]).to_s %>' + :mailcollector: + :server: 'imap.server.de' + :mail_address: "service@mail" + :password: "password" + :aliases: + - 'alias_one@kit.edu' + - 'alias_two@kit.edu' + production: diff --git a/config/initializers/datacollectors.rb b/config/initializers/datacollectors.rb index 54efbebcd7..726c34a362 100644 --- a/config/initializers/datacollectors.rb +++ b/config/initializers/datacollectors.rb @@ -1,18 +1,17 @@ # frozen_string_literal: true -# set default value -Rails.application.configure { config.datacollectors = nil } +# Generic initialization +service = File.basename(__FILE__, '.rb').to_sym # Service name +service_setter = :"#{service}=" # Service setter +ref = "Initializing #{service}:" # Message prefix -if File.file?(Rails.root.join('config/datacollectors.yml')) - datacollectors_config = Rails.application.config_for :datacollectors - if datacollectors_config - Rails.application.configure do - config.datacollectors = ActiveSupport::OrderedOptions.new - config.datacollectors.services = datacollectors_config[:services] - config.datacollectors.mailcollector = datacollectors_config[:mailcollector] - config.datacollectors.sftpusers = datacollectors_config[:sftpusers] - config.datacollectors.localcollectors = datacollectors_config[:localcollectors] - config.datacollectors.keydir = datacollectors_config[:keydir] - end - end +Rails.application.configure do + config.send(service_setter, config_for(service)) # Load config/.yml +# Rescue: +# - RuntimeError is raised if the file is not found +# - NoMethodError is raised if the yml file cannot be parsed +rescue RuntimeError, NoMethodError, ArgumentError, URI::InvalidURIError => e + Rails.logger.warn "#{ref} Error while loading configuration #{e.message}" + # Create service key or clear config + config.send(service_setter, nil) end diff --git a/config/initializers/datamailcollector.rb b/config/initializers/datamailcollector.rb deleted file mode 100644 index fe6775cbf3..0000000000 --- a/config/initializers/datamailcollector.rb +++ /dev/null @@ -1,12 +0,0 @@ -if File.exist? Rails.root.join('config', 'datamailcollector.yml') - datamailcollector_config = Rails.application.config_for :datamailcollector - - Rails.application.configure do - config.datamailcollector = ActiveSupport::OrderedOptions.new - config.datamailcollector.server = datamailcollector_config[:server] - config.datamailcollector.port = datamailcollector_config[:port] - config.datamailcollector.ssl = datamailcollector_config[:ssl] - config.datamailcollector.mail_address = datamailcollector_config[:mail_address] - config.datamailcollector.password = datamailcollector_config[:password] - end -end diff --git a/db/schema.rb b/db/schema.rb index 12ab0a0f71..44ff56fa90 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_07_11_120833) do +ActiveRecord::Schema.define(version: 2024_07_26_064022) do # These are extensions that must be enabled in order to support this database enable_extension "hstore" @@ -1778,4 +1778,4 @@ users WHERE ((channels.id = messages.channel_id) AND (messages.id = notifications.message_id) AND (users.id = messages.created_by)); SQL -end \ No newline at end of file +end diff --git a/db/seeds/development/01_devices.seed.rb b/db/seeds/development/01_devices.seed.rb index 580e860b67..6096263756 100644 --- a/db/seeds/development/01_devices.seed.rb +++ b/db/seeds/development/01_devices.seed.rb @@ -15,81 +15,16 @@ FileUtils.cp_r(config_file_example, config_path) Rails.configuration.datacollectors = Rails.application.config_for(config_name) FileUtils.rm(config_path) -dirname = Rails.root.join(Rails.configuration.datacollectors.dig(:localcollectors, 0, :path)) -# device specific collector dirs: -DIR1 = dirname.join('device1').to_s -DIR2 = dirname.join('device2').to_s +require 'factory_bot' +require 'faker' +require_relative '../../../spec/factories/devices.rb' +require_relative '../../../spec/factories/collector_datafiles.rb' -DEVICE_SEEDS = [ - { - name_abbreviation: 'D1', - name: 'New Device 1', - first_name: '1-Dev', - last_name: 'Ice', - email: 'device1@email.de', - datacollector_method: 'filewatcherlocal', - datacollector_dir: DIR1, - datacollector_host: nil, - datacollector_user: nil, - datacollector_authentication: nil, - datacollector_number_of_files: 1, - datacollector_key_name: nil, - datacollector_user_level_selected: false, - }, - { - name_abbreviation: 'D2', - name: 'New Device 2', - first_name: '2-Dev', - last_name: 'Ice', - email: 'device2@email.de', - datacollector_method: 'folderwatcherlocal', - datacollector_dir: DIR2, - datacollector_host: nil, - datacollector_user: nil, - datacollector_authentication: nil, - datacollector_number_of_files: 0, - datacollector_key_name: nil, - datacollector_user_level_selected: false, - }, -] - - -def create_collector_folders - [DIR1, DIR2].each do |dir| - FileUtils.mkdir_p(dir) unless File.directory?(dir) - end -end - -def touch_files - Person.pluck(:name_abbreviation).each do |na| - # create dummy data file for Dv1 - file collection - file = Pathname.new(DIR1).join("#{na}-#{Time.now.to_i}") - FileUtils.touch(file) - - # create dummy folder with 1 file for Dv1 - folder collection - dir = Pathname.new(DIR2).join("#{na}-#{Time.now.to_i}") - file = dir.join('dummy') - FileUtils.mkdir_p(dir) - FileUtils.touch(file) - end -end - - -def find_or_create_device - DEVICE_SEEDS.each do |seed| - message = "#########################\n Seeding device #{seed[:name_abbreviation]}" - Rails.logger.debug { "#{message}..." } - Device.find_by(name_abbreviation: seed[:name_abbreviation]) && Rails.logger.debug("#{message}: already existing") && next - Device.with_deleted.find_by(name_abbreviation: seed[:name_abbreviation]) && - Rails.logger.debug("#{message}: already existing but soft-deleted") && next - Device.create!(seed) && Rails.logger.debug("#{message}: created") - end +start_sequence = ActiveRecord::Base.connection.execute("SELECT last_value FROM devices_id_seq;").first.fetch('last_value', nil) +[:file_local, :folder_local].each do |trait| + FactoryBot.create(:device, trait, start_sequence: start_sequence) +rescue ActiveRecord::RecordInvalid => e + puts "Device already exists: #{e.record.name}" end -# create collector folders for the Device validation to pass -create_collector_folders -# find or create devices -find_or_create_device -# touch files for the devices to be collected -touch_files diff --git a/db/seeds/development/02_datacollector.seed.rb b/db/seeds/development/02_datacollector.seed.rb new file mode 100644 index 0000000000..3e004a08c8 --- /dev/null +++ b/db/seeds/development/02_datacollector.seed.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# Device Model Seeds +# will seed two devices with local-file and local-folder collectors respectively. +# Idempotent: will not create a device if it already exists. +# but will create collector folders for each device and touch files for each user. + +# Prepare the collector root directory according to the datacollectors example config +# copy, load, and remove the example config to set up the collector root directory temporarily +# So that the devices can be seeded. +# NB seeded devices cannot be updated if the collector root directory config is not set up accordingly +config_file_example = Rails.root.join('config/datacollectors.yml.example') +config_name = :tmp_datacollectors +config_path = Rails.root.join("config/#{config_name}.yml") +FileUtils.cp_r(config_file_example, config_path) +Rails.configuration.datacollectors = Rails.application.config_for(config_name) +FileUtils.rm(config_path) + +require 'factory_bot' +require 'faker' +require_relative '../../../spec/factories/devices.rb' +require_relative '../../../spec/factories/collector_datafiles.rb' + +# build dummy (empty) data files for a set of users and devices +# the data can be collected by the datacollector and moved to the correponding +# user's Inbox +name_abbrs = Person.limit(50).pluck(:name_abbreviation) +Device.where.not(datacollector_method: nil).limit(50).each do |device| + FactoryBot.build(:data_for_collector, device: device, user_identifiers: name_abbrs) +end + diff --git a/db/seeds/development/02_vessels.seed.rb b/db/seeds/development/03_vessels.seed.rb similarity index 100% rename from db/seeds/development/02_vessels.seed.rb rename to db/seeds/development/03_vessels.seed.rb diff --git a/lib/datacollector/collector.rb b/lib/datacollector/collector.rb new file mode 100644 index 0000000000..32cb5e2488 --- /dev/null +++ b/lib/datacollector/collector.rb @@ -0,0 +1,248 @@ +# frozen_string_literal: true + +# @todo: move config logic to config class and DRY validation from AR Device model +module Datacollector + # File and Folder Collector: + # Inspects the Devices' folders for new files and collect them to ELN user account if possible + # + # @!attribute [r] device + # @return [Device] AR Device + # @!attribute [r] config + # @return [Datacollector::Configuration] The configuration for the device + # + class Collector + # Execute the collection for devices + # @param devices [Array] the devices to collect data from + # @option allowed_methods [Array] only allow collection for given methods + def self.bulk_execute(devices, allowed_methods: []) + logger = DCLogger.new(__method__) + devices.each do |device| + new(device).execute(allowed_methods: allowed_methods) + rescue Errors::DatacollectorError, SFTPClientError => e + logger.error(device.info, e.message) + end + end + + attr_reader :device, :config + + # Initialize the collector with a device + # + # @param [Device] an object responding to the following methods: + # - datacollector_method (String) the method to use for data collection (see COLLECTOR_METHODS) + # - datacollector_dir (String) the directory path to inspect + # - datacollector_user_level_selected (Boolean) whether to inspect the user level folders + # - datacollector_number_of_files (Integer) the number of files to collect (folderwatcher only) + # - datacollector_host (String) the host to connect to over sftp (sftp watcher only) + # - datacollector_user (String) the user to connect to the SFTP server (sftp watcher only) + # - datacollector_authentication (String) the authentication method to use + # (password or keyfile - sftp watcher only) + # - datacollector_key_name (String) file name of the key file to use (sftp watcher only - keyfile authentication) + # - info (String) additional information about the device + def initialize(device) + @device = device + @config = Configuration.new!(device) + end + + # config aliases + delegate :collector_dir, to: :config + delegate :expected_count, to: :config + delegate :file_collector?, to: :config + delegate :log, to: :config + delegate :sftp, to: :config + + ################################################################## + # Collector main execution + ################################################################## + + # Executes the appropriate method based on the use_sftp attribute + # + # @param method [String] skip execution if the provided method is not the same as the device's method + def execute(allowed_methods: []) + return if allowed_methods.present? && allowed_methods.exclude?(method) + + device.datacollector_user_level_selected ? inspect_user_folders : inspect_folder + end + + private + + ################################################################## + # Inspection and collection methods + ################################################################## + + # Build the Correspondence object based on a device and a path + # + # @param device [Device] the device sending the file + # @param path [Datacollector::CollectorFile] the collector file + # @param delete [Boolean] whether to delete the file or dir IF no instance of Correspondence is returned + # @return [Datacollector::Correspondence] the Correspondence object + def correspondence_by_file(device, file) + Correspondence.new(device, file.path) + rescue Errors::DatacollectorError => e + log.info(__method__, e.message) + false + end + + # Inspects user folders for new folders and processes them. + # The directory should have the following structure: + # path/to/device/datacollector_dir + # | + # |__user1_identifier + # | |__ file_or_dir1 + # | |__ file_or_dir2 + # | + # |__user2_identifier + # | |__ file_or_dir3 + # | |__ file_or_dir4 + # | ... + # @param device [Device] the device to inspect + # @note user folders are expected to be at the top level of the collector directory + # and are not deleted after processing even if no correspondence is found + def inspect_user_folders + new_folders(collector_dir).each do |user_dir| + correspondence = correspondence_by_file(device, user_dir) + next unless correspondence + + inspect_folder(user_dir.to_s, correspondence: correspondence) + end + end + + # Inspects the folder for new folders and processes them + ## The directory should have the following structure if no correspondence is provided: + # path/to/device/datacollector_dir + # | + # |__ user1_identifier-file_or_dir1 + # |__ user1_identifier-file_or_dir2 + # |__ user2_identifier-file_or_dir3 + # |__ user2_identifier-file_or_dir4 + # .... + # @param dir [String] the directory path to inspect + # @param correspondence [Datacollector::Correspondence] the correspondence collector + # @note folders/files are expected to be at the top level of the collector directory + # and are deleted after processing whether a correspondence is found or not + def inspect_folder(dir = collector_dir, correspondence: nil) + new_files_or_folders(dir).each do |file| + next if previous_failure?(file) + next unless ready?(file) + + current_correspondence = correspondence || correspondence_by_file(device, file) + attach_file_or_folder(current_correspondence, file) + rescue StandardError => e + log.error(e.message, e.backtrace.join('\n')) + end + end + + # Define which attacher to use and use it + # + # @param (@see #attach_file) or (@see #attach_folder) + def attach_file_or_folder(correspondence, file) + if correspondence + file.directory? ? attach_folder(correspondence, file) : attach_file(correspondence, file) + end + try_delete_or_create_error(file) + end + + # Find top level files or folders in the directory depending on the collector method + # + # @param dir [String] the directory to inspect, defaults to the device's directory + # @return [Array] the files or folders + def new_files_or_folders(dir = nil) + file_collector? ? new_files(dir) : new_folders(dir) + end + + # Retrieves new directories in the monitored path. + # + # @param dir_path [String] The path to the directory being monitored. + # @return [Array] An array of new directory paths found in the specified directory. + def new_folders(dir) + CollectorFile.entries(dir, dir_only: true, top_level: true, sftp: @sftp) || [] + end + + # Gets the new files in the monitored folder + # + # @return [Array] the list of files + def new_files(dir) + CollectorFile.entries(dir, file_only: true, top_level: true, sftp: @sftp) + &.reject { |file| file.to_s.end_with?('.filepart', '.part') } || [] + end + + # Collect and create an attachment of the folder: + # Zip the folder locally and attach the zip file and + # try to delete the original folder after attaching the zip file. + # + # @param correspondence [Datacollector::Correspondence] the collector correspondence + # @param folder [Datacollector::CollectorFile] the folder to attach + def attach_folder(correspondence, folder) + tmp_zip = folder.to_zip + name = folder.name + correspondence.attach("#{name}.zip", tmp_zip.path, name) + log.info('Stored', name) + ensure + cleanup_tempfile(tmp_zip) if tmp_zip + end + + # Collect and create an attachment of the file: + # Download the file locally if remote and attach it, then + # try to delete the original file after attaching it. + # + # @param correspondence [Datacollector::Correspondence] the collector correspondence + # @param folder [Datacollector::CollectorFile] the file to attach + def attach_file(correspondence, file) + local_file = file.as_local_file + name = file.name + correspondence.attach(name, local_file.path) + log.info('Stored', name) + ensure + cleanup_tempfile(local_file) if local_file + end + + # Clean up the tempfile if it exists + # @param file_data [String, Tempfile] The file content or tempfile + def cleanup_tempfile(file) + return unless file.is_a?(Tempfile) + + file.close + file.unlink + end + + def try_delete(fstruct) + fstruct.delete! + rescue StandardError => e + log.error(__method__, "#{fstruct.path} >>> #{e.message}") + false + end + + def try_delete_or_create_error(file) + try_delete(file) || CollectorError.find_or_create_by_path(file.path, file.mtime) + end + + def previous_failure?(file) + return false unless CollectorError.find_by_path(file.path, file.mtime) + + try_delete(file) + true + end + + # Get the modification time difference for the folder/file offset by the wait time for the dc method + # @param [CollectorFile] + # @return [Integer] the modification time difference in seconds (positive if the folder/ is ready for collection) + def modification_time_diff(folder) + Time.zone.now - folder.mtime - config.sleep_time + end + + # Wait some time before if dir just created and no fixed number of files expected + def ready?(path) + return log.info(__method__, 'Folder not ready!') && false unless modification_time_diff(path).positive? + return true if expected_count.zero? || file_collector? + + path.directory? && correct_file_count?(path.file_count) + end + + # Check if the folder is ready for collection + # compare the number of files in the folder with the expected number set in the device config + def correct_file_count?(file_count) + return true if expected_count.zero? + + file_count == expected_count || (log.info(__method__, 'Wrong number of files!') && false) + end + end +end diff --git a/lib/datacollector/collector_file.rb b/lib/datacollector/collector_file.rb new file mode 100644 index 0000000000..66e9922a0a --- /dev/null +++ b/lib/datacollector/collector_file.rb @@ -0,0 +1,221 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'net/sftp' + +module Datacollector + # Class file utility wrapper for handling file/folder operations + # whether the source is on local fs or remote through sftp + # + # + # @!attribute [r] pathname + # @return [Pathname] The path of the file/folder as a Pathname object + # @!attribute [r] path + # @return [String] The path of the file/folder as a string + # @!attribute [r] relative_path + # @return [String] The relative path input of the file/folder as a string + # @!attribute [r] root_path + # @return [String] The root directory path input as a string + # + class CollectorFile + attr_reader :pathname, :path, :relative_path, :root_path + + def self.new!(...) + new(...).validate! + end + + def self.entries(path, top_level: true, file_only: false, dir_only: false, sftp: nil) + new(path, sftp: sftp).entries_as(top_level: top_level, file_only: file_only, dir_only: dir_only) + end + + # Initialize a new CollectorFile object + # + # @param relative_path [String] The relative path of the file or directory structure + # @param root_path [String] The root directory path of the file is relative from + # @option sftp [SFTPClient] nil if the file/folder is on local fs or an SFTPClient if the file is on an SFTP server + # @option mtime [Time] nil if the attribute should be fetched from the file system, + # otherwise initialized with the given value + # @return [FStruct] The FStruct object + def initialize(relative_path, root_path = nil, sftp: nil, mtime: nil) + @root_path = root_path.to_s + @relative_path = relative_path.to_s + @sftp = sftp + @mtime = mtime + @pathname = Pathname.new(@root_path).join(@relative_path) + @path = @pathname.to_s + end + + def validate! + raise ArgumentError, 'not a valid SFTP Client' if @sftp.present? && !sftp? + raise ArgumentError, "The combined path '#{path}' does not exist." unless exist? + + self + end + + # FileUtils like methods + + def delete + delete! + rescue StandardError + false + end + + def delete! + file? ? delete_file : delete_folder + true + end + + # Delete a file + def delete_file + sftp? ? @sftp.remove_file!(@path) : @pathname.delete + true + end + + # Delete the folder + def delete_folder + sftp? ? @sftp.remove_dir!(@path) : rmtree + end + + # @return [Time] The mtime of the file + def mtime + @mtime ||= sftp? ? Time.zone.at(@sftp.mtime(@path)) : @pathname.mtime + end + + # Returns an array of file/folder in the @path directory + # + # @param filter [string] The glob filter to apply + # @option file_only [Boolean] if true, only files should be returned + # @option dir_only [Boolean] if true, only directories are returned + # @return [Array, Array] The array of file/folders in the directory + def typed_glob(filter, dir_only: false, file_only: false) + return glob(filter).select(&:file?) if file_only + return glob(filter).select(&:directory?) if dir_only + + glob(filter) + end + + # Recursively count the number of files in the folder + # + # @return [Integer] The total number of files in the folder and its subfolders + def file_count + typed_glob('**/*', file_only: true).count + end + + # Returns an array of CollectorFile file/folder from @path directory + # + # @option file_only [Boolean] if true, only files should be returned + # @option dir_only [Boolean] if true, only directories are returned + # @option top_level [Boolean] if true, only top_level entries are returned + # @return [Array] The array of file/folders in the directory + def entries_as(dir_only: false, file_only: false, top_level: true) + filter = top_level ? '*' : '**/*' + typed_glob(filter, file_only: file_only, dir_only: dir_only).map do |entry| + self.class.new( + build_relative_path(entry), path, sftp: @sftp, mtime: fetch_mtime(entry) + ) + end + end + + # Basename alias + # @return [String] The name of the file + def name + basename + end + + # Path alias + # @return [String] The path of the File + def to_s + path + end + + # Zip the folder and its content into a tmp file + # @return [Tempfile] The zip file + def to_zip + return if file? + + tmp_zip = Tempfile.new + options = { top_level: false, file_only: true } + Dir.mktmpdir do |tmpdir| + @sftp.download_directory!(@path, tmpdir) if sftp? + ::Zip::File.open(tmp_zip.path, ::Zip::File::CREATE) do |zipfile| + (sftp? ? CollectorFile.entries(tmpdir, **options) : entries_as(**options)).each do |entry| + zipfile.add(entry.relative_path, entry.path) + end + end + end + tmp_zip.rewind + tmp_zip + end + + # @return [File, Tempfile] The file object + def as_local_file + return File.new(@pathname) unless sftp? + + tmpfile = Tempfile.new + @sftp.download_file!(@path, tmpfile.path) + tmpfile.rewind + tmpfile + end + + private + + # Get the mtime of the file from an item of entries + # + # @param entry [Pathname, Net::SFTP::Protocol::V01::Name] The entry to get the mtime from + # @return [Integer] The mtime of the file + def fetch_mtime(entry) + case entry + when Pathname + entry.mtime + when Net::SFTP::Protocol::V01::Name + Time.zone.at(entry.attributes.mtime) + end + end + + # Build fstruct.relative_path an item of entries + # + # @param entry [Pathname, Net::SFTP::Protocol::V01::Name] The entry to get the relative_path of + # @return [Pathname] The relative path of the file + def build_relative_path(entry) + case entry + when Pathname + entry.relative_path_from(@pathname).to_s + when Net::SFTP::Protocol::V01::Name + entry.name + end + end + + # Delegate methods to the sftp or pathname object + # + # @note the following methods are defined: + # :basename @return [String] The basename of the file/folder + # :dirname @return [String] The dirname of the file/folder + # :extname @return [String] The extension of the file + # :exist? @return [Boolean] True if the file/folder exists + # :file? @return [Boolean] True if the file is a file + # :directory? @return [Boolean] True if the file is a directory + # :to_s @return [String] The path of the file/folder + def method_missing(method, *args, &block) + case method + when :basename, :dirname, :extname + @pathname.send(method, *args, &block).to_s + else + sftp? ? @sftp.send(method, @path, *args, &block) : @pathname.send(method, *args, &block) + end + end + + def respond_to_missing?(method, include_private = false) + if sftp? + @sftp.respond_to?(method, include_private) + else + @pathname.respond_to?(method, include_private) + end + end + + # test sftp object + # @return [Boolean] True if the sftp object is a Net::SFTP::Session + def sftp? + @sftp.is_a?(SFTPClient) + end + end +end diff --git a/lib/datacollector/collector_helper.rb b/lib/datacollector/collector_helper.rb deleted file mode 100644 index 253f9109bf..0000000000 --- a/lib/datacollector/collector_helper.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -class CollectorHelper - attr_reader :sender, :sender_container, :recipient - - def initialize(from, cc = nil) - if (from.is_a?(Device) && cc.is_a?(User)) || (from.is_a?(User) && from == cc) - @sender = from - @recipient = cc - prepare_containers - elsif cc - @sender = Device.find_by email: from - @sender ||= Device.find_by email: from.downcase - @recipient = User.find_by email: cc - @recipient ||= User.find_by email: cc.downcase - prepare_containers if @sender && @recipient - else - @sender = User.find_by email: from - @sender ||= User.find_by email: from.downcase - @recipient = @sender - prepare_containers if @sender - end - end - - def sender_recipient_known? - (@sender && @recipient) - end - - def prepare_new_dataset(subject) - return nil unless sender_recipient_known? - - Container.create( - name: subject, - container_type: 'dataset', - parent: @sender_container, - ) - end - - def prepare_dataset(subject) - return nil unless sender_recipient_known? - - Container.where( - name: subject, - container_type: 'dataset', - parent: @sender_container, - ).first_or_create - end - - def self.hash(dir_or_file, sftp) - if sftp - sftp.file.open(dir_or_file, 'r') do |d| - date = d.stat.mtime - Digest::SHA256.hexdigest(dir_or_file + date.to_s) - end - else - date = File.mtime(dir_or_file).strftime('%Y%m%d%H%M') - Digest::SHA256.hexdigest(File.new(dir_or_file).path + date) - end - end - - def self.write_error(e) - error = CollectorError.new(error_code: e) - error.save! unless CollectorError.find_by error_code: e - end - - private - - def prepare_containers - unless @recipient.container - @recipient.container = Container.create( - name: 'inbox', - container_type: 'root', - ) - end - sender_box_id = "sender_box_#{@sender.id}" - @sender_container = Container.where( - name: @sender.name, - container_type: sender_box_id, - parent_id: @recipient.container.id, - ).first_or_create - end -end diff --git a/lib/datacollector/collector_helper_set.rb b/lib/datacollector/collector_helper_set.rb deleted file mode 100644 index 0ffefc25fe..0000000000 --- a/lib/datacollector/collector_helper_set.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class CollectorHelperSet - attr_reader :helper_set - - def initialize(from, cc_list) - @helper_set = [] - cc_list.each do |cc| - h = CollectorHelper.new(from, cc) - @helper_set.push(h) if h.sender_recipient_known? - end - end - - def sender_recipient_known? - @helper_set.length.positive? - end -end diff --git a/lib/datacollector/configuration.rb b/lib/datacollector/configuration.rb new file mode 100644 index 0000000000..697749cb8f --- /dev/null +++ b/lib/datacollector/configuration.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +# @todo: DRY validation from AR Device model +module Datacollector + # Configuration class for the Datacollector + # @note: This class is used to validate the configuration of the datacollector + class Configuration + # Collectors Method types and description + COLLECTOR_METHODS_HASH = { + 'filewatchersftp' => 'file watcher over sftp', + 'folderwatchersftp' => 'folder watcher over sftp', + 'filewatcherlocal' => 'file watcher over local fs', + 'folderwatcherlocal' => 'folder watcher over local fs', + # 'mailcollector' => 'mail collector', + }.freeze + + # (see COLLECTOR_METHODS_HASH) + COLLECTOR_METHODS = COLLECTOR_METHODS_HASH.keys.freeze + + # @return [OrderedOptions] the datacollector configuration + def self.config + Rails.configuration.datacollectors + end + + # Validate the configuration and return a new config instance given a device + # + # @return [Configuration] the configuration object + def self.new!(...) + new(...).validate + end + + # Get the sleep time for the watcher + # + # @param method [String] the method to get the sleep time for + + # @return [Integer] the sleep seconds, default is 5 + def self.sleep_time(method = nil) + return 20 if method.blank? + + config.services&.find { |service| service[:name] == method } + &.dig(:watcher_sleep)&.to_i || 20 + end + + # @return [Device] the device to collect from + attr_reader :device, :log + + # Initialize the collector with a device + # + # @param [Device] an object responding to the following methods: + # - datacollector_method (String) the method to use for data collection (see COLLECTOR_METHODS) + # - datacollector_dir (String) the directory path to inspect + # - datacollector_user_level_selected (Boolean) whether to inspect the user level folders + # - datacollector_number_of_files (Integer) the number of files to collect (folderwatcher only) + # - datacollector_host (String) the host to connect to over sftp (sftp watcher only) + # - datacollector_user (String) the user to connect to the SFTP server (sftp watcher only) + # - datacollector_authentication (String) the authentication method to use + # (password or keyfile - sftp watcher only) + # - datacollector_key_name (String) file name of the key file to use (sftp watcher only - keyfile authentication) + # - info (String) additional information about the device + def initialize(device) + @device = device + @log = DCLogger.new(device.info) + @sftp = nil + end + + def validate + info = device.info + raise Errors::DatacollectorError, 'No datacollector system configuration!' if config.blank? + raise Errors::DatacollectorError, "Invalid collector-method! #{info}" if collector_method.blank? + raise Errors::DatacollectorError, "No openable SFTP client for #{info}!" if sftp_collector? && !sftp.open? + raise Errors::DatacollectorError, "Not a valid sftp dir for #{info}!" unless valid_sftp_dir?(collector_dir) + raise Errors::DatacollectorError, "Not a valid local dir for #{info}!" unless valid_local_dir?(collector_dir) + + self + end + + ################################################################## + # some device attributes aliases + ################################################################## + + # @return [OrderedOptions] the datacollector configuration + def config + @config ||= self.class.config + end + + def collector_dir + device.datacollector_dir.to_s + end + + def expected_count + device.datacollector_number_of_files.to_i + end + + # The current collector method + # + # @return [String] the collector method + def collector_method + @collector_method ||= COLLECTOR_METHODS.find { |m| m == device.datacollector_method } + end + + # Sleep time for the watcher + # @return [Integer] the sleep time in seconds + # @note: default is 5 seconds + def sleep_time + self.class.sleep_time(collector_method) + end + + ################################################################## + # collector_method validations + ################################################################## + + # @return [Boolean] whether the collector method is on local file system + def local_collector? + collector_method.end_with?('local') + end + + # @return [Boolean] whether the collector method is over sftp + def sftp_collector? + collector_method.end_with?('sftp') + end + + # @return [Boolean] whether the collector method is for file + def file_collector? + collector_method.start_with?('file') + end + + # Validate for sftp collector that dir is accessible + # + # @param dir [String] the directory to validate + # @return [Boolean] whether the directory is allowed + def valid_sftp_dir?(dir) + return true unless sftp_collector? + + CollectorFile.new!(dir, sftp: sftp) && true + rescue ArgumentError + false + end + + # Validate for local collector that dir is in the allowed directories list as defined in the configuration + # + # @param dir [String] the directory to validate + # @return [Boolean] whether the directory is allowed + def valid_local_dir?(dir) + return true unless local_collector? + + config.localcollectors.any? do |local| + root = Pathname(local[:path]).realpath.to_s + Pathname(dir).realpath.to_s.start_with?(root) + end + rescue Errno::ENOENT + false + end + + # Get the SFTPClient options for the device + # + # @param device [Device] the device to get SFTP arguments for + # @return [Hash] the SFTP arguments + def sftp_options_for_device + options = {} + case device.datacollector_authentication + when 'keyfile' + options[:keys] = key_path_for_device + options[:keys_only] = true + when 'password' + options[:password] = password_for_device + else + log.info(__method__, 'connection method is unknown!') + end + + if options[:password].blank? && options[:keys].blank? + log.info(__method__, 'No valid host or no password or key file found!') + return {} + end + options + end + + # Get the password for the device + # + # @return [String] the password + def password_for_device + config.sftpusers&.find { |entry| entry[:user] == device.datacollector_user }&.fetch(:password, nil) + end + + # Get the key path for the device + # + # @return [String] the absolute path to the key + def key_path_for_device + key_dir = Pathname.new(config.keydir) + key_dir = Rails.root.join(key_dir) if key_dir.relative? + key_name = Pathname.new(device.datacollector_key_name) + key_name = key_dir.join(key_name) if key_name.relative? + # confirm key path is in the keydir + key_name.realpath.to_s.start_with?(key_dir.realpath.to_s) ? key_name.to_s : nil + end + + ################################################################## + ## SFTP + ################################################################## + + # Initialize the SFTP client for the device + # + # @param device [Device] the device to initialize the SFTP client for + # @return [SFTPClient] the SFTP client + def sftp + return unless sftp_collector? + + @sftp ||= SFTPClient.new( + device.datacollector_host, + device.datacollector_user, + **sftp_options_for_device, + ) + end + end +end diff --git a/lib/datacollector/correspondence.rb b/lib/datacollector/correspondence.rb new file mode 100644 index 0000000000..4ca535b27a --- /dev/null +++ b/lib/datacollector/correspondence.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require_relative 'correspondence_helpers' + +module Datacollector + # Define the parties (Users, Devices) involved in one exchange and the destination container + # Create and attach the attachment from file path to the proper Inbox Container (model) of the recipient + # @!attribute [r] sender + # @return [Device, User] the sender of the correspondence + # @note the sender can be a device or a user! + # @!attribute [r] recipient + # @return [User] the recipient of the correspondence + # @note the recipient can only be a user (Person or Group)! + # @!attribute [r] sender_container + # @return [Container] the sender's container in the receiver's inbox + # @note the sender's container is the destination container for the correspondence + # User (receiver) -> Container 'Inbox' -> Container named after the sender (sender_container) + # -> Attachments or Container(datasets) + class Correspondence + include CorrespondenceHelpers + extend CorrespondenceHelpers + + attr_reader :sender, :sender_container, :recipient + + # @param to [String, User, Nil] The info to determine the recipient (User) or the recipient itself + # @param from [String, User, Device] The info to determine the sender (User Device) or the sender itself + # @return [Correspondence] The correspondence object + # @raise [Error::DataCollectorError] If no valid sender or recipient is found + def initialize(from, to) + # if param is a string, find the user or device + @sender = from.is_a?(String) ? find_sender(from) : from + @recipient = to.is_a?(String) ? find_recipient(to) : to + + # if the sender is a user (not a device), then it can only send to its own account + @recipient = @sender if @sender.is_a?(User) + + raise Errors::DatacollectorError, "Sender not found #{from}" unless validate(@sender) + raise Errors::DatacollectorError, "Recipient not found #{to}" unless validate(@recipient) + + prepare_containers + end + + # Attach the file to the appropriate dataset and trigger notifications + # @param file_name [String] The name of the file + # @param file_path [String,Path] The path to the file + # @return [Attachment] The updated attachment + def attach(filename, file_path, dataset_name = Time.zone.now.strftime('%Y-%m-%d')) + attachment = create_attachment(filename, file_path) + dataset = prepare_dataset(dataset_name) + # rubocop:disable Rails/SkipsModelValidations + dataset.touch + # rubocop:enable Rails/SkipsModelValidations + attachment.update!(attachable: dataset) + + # Add notifications + queue = "inbox_#{sender.id}_#{recipient.id}" + schedule_notification(queue) unless Delayed::Job.find_by(queue: queue) + + attachment.present? + end + + private + + # Find or create the container where the attachments will be associated + # @return [Container] The container where the attachments will be associated + # @note If the recipient has no Inbox container, it will be created + # @note The sender container will be attached to the user's inbox + def prepare_containers + # if the recipient has no inbox, create one + unless recipient.container + recipient.container = Container.create( + name: 'inbox', + container_type: 'root', + ) + end + @sender_container = Container.where( + name: sender.name, + container_type: build_sender_box_id(sender.id), + parent_id: @recipient.container.id, + ).first_or_create + end + + # Build the sender box id + # @param sender_id [Integer] The sender id + # @return [String] The sender box id + def build_sender_box_id(sender_id) + "sender_box_#{sender_id}" + end + + # Create an attachment for the given file data + # @param name [String] The filenameFor a user + # @param path [String, Path] The file path + # @return [Attachment] The created attachment + def create_attachment(name, file_data) + Attachment.create( + filename: name, + created_by: sender.id, + created_for: recipient.id, + file_path: file_data, + ) + end + + # Prepare the dataset that will be used to store the attachments + # @param subject [String] The name of the dataset + # @todo: limit the number of attachments in the dataset to 50 due to pagination on the inbox + def prepare_dataset(subject) + container = Container.where( + name: subject, + container_type: 'dataset', + parent: sender_container, + ).first_or_create + # return the container if it has less than Entity::InboxEntity::MAX_ATTACHMENTS + return container if container.attachments.count < 50 # Entity::InboxEntity::MAX_ATTACHMENTS + + # add a counter `_02` to the subject or increment the subject counter and try again + subject = /(.+)_(\d+)$/.match?(subject) ? subject.next : "#{subject}_02" + prepare_dataset(subject) + end + + # Schedule a notification job + # @param queue [String] The queue name + def schedule_notification(queue) + MessageIncomingDataJob.set(queue: queue, wait: 3.minutes).perform_later( + sender_container.name, sender.id, recipient.id + ) + end + end +end diff --git a/lib/datacollector/correspondence_array.rb b/lib/datacollector/correspondence_array.rb new file mode 100644 index 0000000000..393dd09e20 --- /dev/null +++ b/lib/datacollector/correspondence_array.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Datacollector + # Build an array of Correspondence object between two arrays + # + # @!attribute [r] sender + # @return [Device, User] AR Device or User + # @!attribute [r] recipients + # @return [Array] Array of recipients for the exchange + # + class CorrespondenceArray < Array + attr_reader :sender, :recipients + + # @param from [String, User, Device] The sender identifier of the exchange + # @param to [Array<[String, User]>] Array of recipient identifiers for the exchange + def initialize(from, to_list) + @sender = from.is_a?(String) ? Correspondence.find_sender(from) : from + errors = [] + raise Errors::DatacollectorError, "Sender not found #{from}" unless Correspondence.validate(@sender) + + correspondences = to_list.filter_map do |receiver| + Correspondence.new(@sender, receiver) + rescue Errors::DatacollectorError => e + errors << e.message + nil + end + raise Errors::DatacollectorError, errors.join("\n") if correspondences.empty? + + super(correspondences) + + @recipients = map(&:recipient) + end + end +end diff --git a/lib/datacollector/correspondence_helpers.rb b/lib/datacollector/correspondence_helpers.rb new file mode 100644 index 0000000000..2cc8bd869a --- /dev/null +++ b/lib/datacollector/correspondence_helpers.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Datacollector + module CorrespondenceHelpers + # try to find device/user from identifier input + # @param identifier [String] input string + # @return [Device, User, Nil] device or user object + # @note the sender can be a device or a user! + # @note finding a device is preferred over finding a user! + # @note that since DEvice are not a subclass of User, we need to check for both + # separately and email/name_abbreviation are not unique for both + def find_sender(identifier) + find_device(identifier) || find_user(identifier) + end + + # try to find the recipient from identifier input + # @param identifier [String] input string + # @return [User, Nil] user + # @note the receiver can only be a user! + # alias_method :find_recipient, :find_user + def find_recipient(identifier) + find_user(identifier) + end + + # Validate the sender or recipient + # @param obj [User, Device, Nil] The object to check + # @return [Boolean] True if the object is an active User or a Device + def validate(obj) + case obj + when Person, Group + obj.account_active + when Device + true + else + false + end + end + + # Find a device by email or name abbreviation + # @param [String] The email or name abbreviation + def find_device(raw_identifier) + identifier = parse_identifier(raw_identifier) + identifier.include?('@') ? Device.by_email(identifier).first : Device.find_by(name_abbreviation: identifier) + end + + # Find a user by email (insensitive case) or name abbreviation + # @param [String] The email or name abbreviation + # @return [User, Nil] The user object (Person or Group) or nil if not found + # @note if case sensitive name abbreviation, the user will not be found + def find_user(raw_identifier) + identifier = parse_identifier(raw_identifier) + scope = User.where(type: %w[Person Group]) + return scope.by_email(identifier).first if identifier.include?('@') + + scope.by_exact_name_abbreviation(identifier).first || scope.by_exact_name_abbreviation(identifier, true).first + end + + private + + # Parse the info string to extract email or name abbreviation + # can be path like: /path/to/email@address or /path/AB-folder-name with AB as name abbreviation + # @note `@` and `-` is not allowed in name abbreviation + # @param [String] The input string + # @return [String] The email or name abbreviation + # @todo: should consider potential customized regex as set in config/user_props.yml + # see User#name_abbreviation_format + def parse_identifier(info) + # remove potential path + info = info.split('/').last.strip + # check if email + info.include?('@') ? info : info.split('-').first.strip + end + end +end diff --git a/lib/datacollector/datacollector_file.rb b/lib/datacollector/datacollector_file.rb deleted file mode 100644 index d42075ae0e..0000000000 --- a/lib/datacollector/datacollector_file.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -# A class as a file data collector -class DatacollectorFile < DatacollectorObject - def collect_from(device) - if @sftp - add_attach_to_container(device, attach_remote(device)) - else - add_attach_to_container(device, attach(device)) - end - end - - def delete - @sftp ? @sftp.remove!(@path) : File.delete(@path) - end - - private - - def attach(device) - att = Attachment.new( - filename: @name, - file_data: File.binread(@path), - content_type: MimeMagic.by_path(@name)&.type, - created_by: device.id, - created_for: recipient.id, - file_path: @path, - ) - att.save! - - att - end - - def attach_remote(device) - begin - tmpfile = Tempfile.new - @sftp.download!(@path, tmpfile.path) - att = Attachment.new( - filename: @name, - file_path: tmpfile.path, - content_type: MimeMagic.by_path(@name)&.type, - created_by: device.id, - created_for: recipient.id, - ) - att.save! - ensure - tmpfile.close - tmpfile.unlink - end - - att - end - - def add_attach_to_container(device, attach, _ = false) - helper = CollectorHelper.new(device, recipient) - dataset = helper.prepare_dataset(Time.zone.now.strftime('%Y-%m-%d')) - attach.update!(attachable: dataset) - - # add notifications - queue = "inbox_#{device.id}_#{recipient.id}" - unless Delayed::Job.find_by(queue: queue) - MessageIncomingDataJob.set(queue: queue, wait: 3.minutes).perform_later( - helper.sender_container.name, helper.sender.id, recipient.id - ) - end - - attach - end -end diff --git a/lib/datacollector/datacollector_folder.rb b/lib/datacollector/datacollector_folder.rb deleted file mode 100644 index bd45980918..0000000000 --- a/lib/datacollector/datacollector_folder.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -class DatacollectorFolder < DatacollectorObject - attr_accessor :files - - def collect(device) - tmpzip = Tempfile.new([@name, '.zip']) - zip_files(tmpzip) - register_new_data(device, tmpzip) - ensure - tmpzip.close - tmpzip.unlink - end - - def delete - if @sftp - sftp.session.exec!("rm -rf #{@path}") - else - FileUtils.rm_r @path - end - end - - private - - def zip_files(tmpzip) - return if @files.nil? - - if @sftp - @files.each do |new_file| - tmpfile = Tempfile.new - sftp.download!(File.join(path, new_file), tmpfile.path) - Zip::File.open(tmpzip.path, Zip::File::CREATE) do |zipfile| - zipfile.add(File.join(@name, new_file), tmpfile.path) - end - ensure - tmpfile.close - tmpfile.unlink - end - else - Zip::File.open(tmpzip.path, Zip::File::CREATE) do |zipfile| - @files.each do |new_file| - zipfile.add(File.join(@name, new_file), File.join(@path, new_file)) - end - end - end - end - - def register_new_data(device, tmpzip) - att = Attachment.new( - filename: "#{@name}.zip", - created_by: device.id, - created_for: recipient.id, - content_type: 'application/zip', - file_path: tmpzip.path, - ) - att.save! - helper = CollectorHelper.new(device, recipient) - dataset = helper.prepare_new_dataset(@name) - att.update!(attachable: dataset) - end -end diff --git a/lib/datacollector/datacollector_object.rb b/lib/datacollector/datacollector_object.rb deleted file mode 100644 index 8e8dceeb9a..0000000000 --- a/lib/datacollector/datacollector_object.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class DatacollectorObject - attr_reader :name, :path, :sftp - - def initialize(path, sftp, recipient_abbr = nil) - @path = path - @recipient_abbr = recipient_abbr || @path.split('/').last.split('-').first - @name = @path.split('/').last - @sftp = sftp - end - - def recipient - User.try_find_by_name_abbreviation @recipient_abbr - end -end diff --git a/lib/datacollector/dc_logger.rb b/lib/datacollector/dc_logger.rb index 002546de9c..ac0f45346d 100644 --- a/lib/datacollector/dc_logger.rb +++ b/lib/datacollector/dc_logger.rb @@ -1,7 +1,22 @@ # frozen_string_literal: true -class DCLogger - def self.log - @@fw_logger ||= Logger.new("#{Rails.root}/log/datacollector.log") +module Datacollector + # Class to log messages to log/datacollector.log + class DCLogger + def initialize(context = 'Datacollector') + @log = Logger.new(Rails.root.join('log/datacollector.log').to_s) + @context = context + @format = lambda do |subcontext, msg = nil| + "#{@context} - #{subcontext} >> #{msg}\n" + end + end + + def info(subcontext, message = nil) + @log.info(@format.call(subcontext, message)) + end + + def error(subcontext, message = nil) + @log.error(@format.call(subcontext, message)) + end end end diff --git a/lib/datacollector/fcollector.rb b/lib/datacollector/fcollector.rb deleted file mode 100644 index 79b42ce08a..0000000000 --- a/lib/datacollector/fcollector.rb +++ /dev/null @@ -1,121 +0,0 @@ -# frozen_string_literal: true - -# File and Folder Collector -class Fcollector - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/MethodLength - # rubocop:disable Metrics/PerceivedComplexity - - def execute(use_sftp) - raise 'No datacollector configuration!' unless Rails.configuration.datacollectors - - unless use_sftp - @sftp = nil - devices(use_sftp).each { |device| inspect_folder(device) } - return - end - - devices(use_sftp).each do |device| # rubocop:disable Metrics/BlockLength - @current_collector = nil - host = device.datacollector_host - uri = URI.parse("ssh://#{host}") - host = uri.host - port = uri.port - - case device.datacollector_authentication - when 'keyfile' - user = device.datacollector_user - kp = key_path(device.datacollector_key_name) - unless kp.file? && kp.exist? - log_info "No key file found <<< #{device.info}" unless kp.file? && kp.exist? - next - end - - args = { - key_data: [], - keys: kp, - auth_methods: %w[publickey], - verbose: :error, - keys_only: true, - } - when 'password', nil - credentials = Rails.configuration.datacollectors.sftpusers.find do |user_attr| - user_attr[:user] == device.datacollector_user - end - unless credentials - log_info("No match user credentials! user: #{device.datacollector_user} >>> #{device.info}") - next - end - user = credentials[:user] - args = { - password: credentials[:password], - } - else - user = nil - args = {} - log_info("connection method is unknown! >>> #{device.info}") - next - end - args[:timeout] = 10 - args[:number_of_password_prompts] = 0 - args[:port] = port if port.present? - - begin - Net::SFTP.start(host, user, **args) do |sftp| - @sftp = sftp - inspect_folder(device) - @sftp = nil - end - rescue StandardError => e - log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") - end - end - end - - # rubocop:enable Metrics/CyclomaticComplexity - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/MethodLength - # rubocop:enable Metrics/PerceivedComplexity - - private - - def devices(use_sftp) - Device.where(datacollector_method: "#{self.class::FCOLL}watcher#{use_sftp ? 'sftp' : 'local'}") - end - - def key_path(key_name) - key_dir = Rails.configuration.datacollectors.keydir - if key_dir.start_with?('/') - Pathname.new(key_dir).join(key_name) - else - Rails.root.join(key_dir, key_name) - end - end - - def log_info(message) - DCLogger.log.info(self.class.name) do - "#{@current_collector&.path} >>> #{message}" - end - end - - def log_error(message) - DCLogger.log.error(self.class.name) do - "#{@current_collector&.path} >>> #{message}" - end - end - - def new_folders(monitored_folder_p) - if @sftp - new_folders_p = @sftp.dir.glob(monitored_folder_p, '*').select( - &:directory? - ) - new_folders_p.map! { |dir| File.join(monitored_folder_p, dir.name) } - else - new_folders_p = Dir.glob(File.join(monitored_folder_p, '*')).select do |e| - File.directory?(e) - end - end - new_folders_p - end -end diff --git a/lib/datacollector/filecollector.rb b/lib/datacollector/filecollector.rb deleted file mode 100644 index ba9d3565e1..0000000000 --- a/lib/datacollector/filecollector.rb +++ /dev/null @@ -1,114 +0,0 @@ -# frozen_string_literal: true - -# Collector: file inspection and collection -class Filecollector < Fcollector - FCOLL = 'file' - - private - - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/MethodLength - # rubocop:disable Metrics/PerceivedComplexity - # rubocop:disable Metrics/BlockLength - - def inspect_folder(device) - directory = device.datacollector_dir - user_level_selected = device.datacollector_user_level_selected - - if user_level_selected - inspect_user_folders(device, directory) - else - new_files(directory).each do |new_file_p| # rubocop:disable Metrics/BlockLength - @current_collector = DatacollectorFile.new(new_file_p, @sftp) - error = CollectorError.find_by error_code: CollectorHelper.hash( - @current_collector.path, - @sftp, - ) - begin - stored = false - if @current_collector.recipient - unless error - @current_collector.collect_from(device) - log_info("Stored! >>> #{device.info}") - stored = true - end - @current_collector.delete - log_info("Status 200 >>> #{device.info}") - else # Recipient unknown - @current_collector.delete - log_info("Recipient unknown. File deleted! >>> #{device.info}") - end - rescue StandardError => e - if stored - CollectorHelper.write_error( - CollectorHelper.hash(@current_collector.path, @sftp), - ) - end - log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") - end - end - end - end - - def new_files(monitored_folder_p) - if @sftp - new_files_p = @sftp.dir.glob(monitored_folder_p, '*').reject( - &:directory? - ) - new_files_p.map! do |f| - File.join(monitored_folder_p, f.name) - end - else - new_files_p = Dir.glob(File.join(monitored_folder_p, '*')).reject do |e| - File.directory?(e) - end - end - new_files_p.delete_if do |f| - f.end_with?('.filepart', '.part') - end - new_files_p - end - - def inspect_user_folders(device, directory) - new_folders(directory).each do |new_folder_p| - recipient_abbr = new_folder_p.split('/').last.split('-').first - recipient = User.try_find_by_name_abbreviation recipient_abbr - - if recipient - new_files(new_folder_p).each do |new_file_p| - @current_collector = DatacollectorFile.new(new_file_p, @sftp, recipient_abbr) - error = CollectorError.find_by error_code: CollectorHelper.hash( - @current_collector.path, - @sftp, - ) - begin - stored = false - - unless error - @current_collector.collect_from(device) - log_info("Stored! >>> #{device.info}") - stored = true - end - - @current_collector.delete - log_info("Status 200 >>> #{device.info}") - rescue StandardError => e - if stored - CollectorHelper.write_error( - CollectorHelper.hash(@current_collector.path, @sftp), - ) - end - log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") - end - end - else # Recipient unknown - log_info("Recipient unknown. >>> #{device.info} >>> #{recipient_abbr}") - end - end - end - - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/MethodLength - # rubocop:enable Metrics/PerceivedComplexity - # rubocop:enable Metrics/BlockLength -end diff --git a/lib/datacollector/foldercollector.rb b/lib/datacollector/foldercollector.rb deleted file mode 100644 index 3760ea9c16..0000000000 --- a/lib/datacollector/foldercollector.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: true - -require 'zip' - -# Collector: folder inspection and collection -class Foldercollector < Fcollector - FCOLL = 'folder' - - private - - def sleep_seconds(device) - 30 || (Rails.configuration.datacollectors.services && - (Rails.configuration.datacollectors.services.find do |e| - e[:name] == device.profile.data['method'] - end || {})[:watcher_sleep]) - end - - def modification_time_diff(device, folder_p) - time_now = Time.zone.now - case device.datacollector_method - when 'folderwatcherlocal' then time_now - File.mtime(folder_p) - when 'folderwatchersftp' - time_now - (Time.zone.at @sftp.file.open(folder_p).stat.attributes[:mtime]) - else 30 - end - end - - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/BlockLength - # rubocop:disable Metrics/MethodLength - # rubocop:disable Metrics/PerceivedComplexity - - def inspect_folder(device) - user_level_selected = device.datacollector_user_level_selected - - if user_level_selected - inspect_user_folders(device) - else - sleep_time = sleep_seconds(device).to_i - new_folders(device.datacollector_dir).each do |new_folder_p| # rubocop:disable Metrics/BlockLength - if (device.datacollector_number_of_files.blank? || device.datacollector_number_of_files.to_i.zero?) && - modification_time_diff(device, new_folder_p) < 30 - sleep sleep_time - end - - @current_collector = DatacollectorFolder.new(new_folder_p, @sftp) - @current_collector.files = list_files - error = CollectorError.find_by error_code: CollectorHelper.hash( - @current_collector.path, - @sftp, - ) - begin - stored = false - if @current_collector.recipient - if device.datacollector_number_of_files.present? && device.datacollector_number_of_files.to_i != 0 && - @current_collector.files.length != device.datacollector_number_of_files.to_i - log_info("Wrong number of files! >>> #{device.info}") - next - end - unless error - @current_collector.collect(device) - log_info("Stored! >>> #{device.info}") - stored = true - end - @current_collector.delete - log_info("Status 200 >>> #{device.info}") - else # Recipient unknown - @current_collector.delete - log_info("Recipient unknown. Folder deleted! >>> #{device.info}") - end - rescue StandardError => e - if stored - CollectorHelper.write_error( - CollectorHelper.hash(@current_collector.path, @sftp), - ) - end - log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") - end - end - end - end - - def list_files - if @sftp - all_files = @sftp.dir.glob(@current_collector.path, '**/*').reject( - &:directory? - ) - all_files.map!(&:name) - else - all_files = Dir.entries(@current_collector.path).reject do |e| - File.directory?(File.join(@current_collector.path, e)) - end - end - all_files.delete_if do |f| - f.end_with?('..', '.', '.filepart', '.part') - end - all_files - end - - def inspect_user_folders(device) - sleep_time = sleep_seconds(device).to_i - new_folders(device.datacollector_dir).each do |new_folder_p| - recipient_abbr = new_folder_p.split('/').last.split('-').first - recipient = User.try_find_by_name_abbreviation recipient_abbr - - if recipient - new_folders(new_folder_p).each do |new_folder| - if (device.datacollector_number_of_files.blank? || device.datacollector_number_of_files.to_i.zero?) && - modification_time_diff(device, new_folder) < 30 - sleep sleep_time - end - - @current_collector = DatacollectorFolder.new(new_folder, @sftp, recipient_abbr) - @current_collector.files = list_files - error = CollectorError.find_by error_code: CollectorHelper.hash( - @current_collector.path, - @sftp, - ) - begin - stored = false - - if device.datacollector_number_of_files.present? && device.datacollector_number_of_files.to_i != 0 && - @current_collector.files.length != device.datacollector_number_of_files.to_i - log_info("Wrong number of files! >>> #{device.info}") - next - end - - unless error - @current_collector.collect(device) - log_info("Stored! >>> #{device.info}") - stored = true - end - - @current_collector.delete - log_info("Status 200 >>> #{device.info}") - rescue StandardError => e - if stored - CollectorHelper.write_error( - CollectorHelper.hash(@current_collector.path, @sftp), - ) - end - log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") - end - end - else # Recipient unknown - log_info("Recipient unknown. >>> #{device.info} >>> #{recipient_abbr}") - end - end - end - - # rubocop:enable Metrics/CyclomaticComplexity - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/BlockLength - # rubocop:enable Metrics/MethodLength - # rubocop:enable Metrics/PerceivedComplexity -end diff --git a/lib/datacollector/mail_configuration.rb b/lib/datacollector/mail_configuration.rb new file mode 100644 index 0000000000..473e6ab7fe --- /dev/null +++ b/lib/datacollector/mail_configuration.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +# @todo: DRY validation from AR Device model +module Datacollector + # Dummy device for mail collector + MailDevice = Struct.new( + 'MailDevice', :datacollector_method, :info, + # unused attributes: + :datacollector_dir, :datacollector_user_level_selected, :datacollector_number_of_files, + :datacollector_host, :datacollector_user, :datacollector_authentication, :datacollector_key_name + ) + + # Configuration class for the Mailcollector + # @note: This class is used to validate the configuration of the mailcollector + class MailConfiguration < Datacollector::Configuration + attr_reader :server, :mail_address, :password, :aliases, :port, :ssl + + # Get the email configuration + # + # @return [Hash] the email configuration + def initialize(device = MailDevice.new('mailcollector', 'Mail Collector')) + super + + mail_config = config&.mailcollector + raise 'No mailcollector configuration!' if mail_config.blank? + + @server = mail_config[:server] + @mail_address = mail_config[:mail_address] + @password = mail_config[:password] + @aliases = mail_config[:aliases] || [] + @port = mail_config.key?(:port) ? mail_config[:port] : 993 + @ssl = mail_config.key?(:ssl) ? mail_config[:ssl] : true + end + end +end diff --git a/lib/datacollector/mailcollector.rb b/lib/datacollector/mailcollector.rb index b6933c06b2..57467ca66c 100644 --- a/lib/datacollector/mailcollector.rb +++ b/lib/datacollector/mailcollector.rb @@ -3,151 +3,199 @@ require 'net/imap' require 'mail' -class Mailcollector - def initialize - raise 'No datacollector configuration!' unless Rails.configuration.datacollectors - - config = Rails.configuration.datacollectors.mailcollector - @server = config[:server] - @mail_address = config[:mail_address] - @password = config[:password] - @aliases = config[:aliases] || [] - @port = config.key?(:port) ? config[:port] : 993 - @ssl = config.key?(:ssl) ? config[:ssl] : true - end +module Datacollector + # Class for collecting attachment data from emails + class Mailcollector + INBOX = 'INBOX' + ENVELOPE = 'ENVELOPE' + RFC822 = 'RFC822' + + attr_reader :config + + def initialize + @config = MailConfiguration.new + login + end - def execute - imap = Net::IMAP.new(@server, @port, @ssl) - response = imap.login(@mail_address, @password) - if response['name'] == 'OK' - log_info('Login...') - imap.select('INBOX') + # Fetches all unseen emails and processes them + # + # @note The process is as follows: + # |__ Fetch all unseen emails + # | |__For each email + # | |__ `open_envelop` : Open the ENVELOPE and set the instance variable @envelope + # | |__ `correspondences` : set the array of Datacollector::Correspondence between sender and recipients + # | | | based on the the email addresses extracted from the envelope (from, to, cc) + # | | |__ `sender_email` + # | | |__ `receivers_emails` + # | | + # | X--> skip to the next email if no correspondence is found + # | | (no ELN user found as recipient or the sender is not an ELN user/device) + # | | + # | |__ `handle_message` Extract and set the message + # | | | + # | | X--> return if message has no attachments + # | | |__ `handle_attachment` for each attachment in the message + # | | |__ `correspondence.attach` create the attachment in the recipient ELN user inbox + # | | + # | |__ delete the email + # | + # |__ Logout + # + def execute imap.search(%w[NOT SEEN]).each do |message_id| - handle_new_mail(message_id, imap) - rescue StandardError => e - log_error e.message + open_envelope(message_id) + next if correspondences.empty? + + handle_message(message_id) + imap.store(message_id, '+FLAGS', [:Deleted]) end - imap.close - else - log_error("Cannot login #{@server}") + rescue StandardError => e + log.error __method__, e.backtrace.join('\n') raise + ensure + logout end - rescue StandardError => e - log_error 'mail collector execute error:' - log_error e.backtrace.join('\n') - raise - ensure - imap.logout - imap.disconnect - end - private - - def handle_new_mail(message_id, imap) - envelope = imap.fetch(message_id, 'ENVELOPE')[0].attr['ENVELOPE'] - raw_message = imap.fetch(message_id, 'RFC822').first.attr['RFC822'] - message = Mail.read_from_string raw_message - helper_set = create_helper_set(envelope) - log_info "Mail from #{message.from}" - unless helper_set - log_info "#{message.from} Email format incorrect or sender unknown!" - return nil + private + + delegate :log, to: :config + + ############################################################################################################ + ## IMAP Connection + ############################################################################################################ + + delegate :server, :mail_address, :password, :aliases, :port, :ssl, to: :config + + # Imap login and navigate to inbox + # (initialize the imap instance variable) + def login + response = imap.login(mail_address, password) + raise unless response['name'] != 'OK' + + log.info('Login...') + imap.select(INBOX) + rescue StandardError => e + log.error("Cannot login #{server}", e.message) + logout end - helper_set.helper_set.each do |helper| - unless helper.sender - log_info "#{message.from} Sender unknown!" - return nil - end - unless helper.recipient - log_info "#{message.from} Recipient unknown!" - return nil - end - if message.attachments - handle_new_message(message, helper) - log_info "#{message.from} Data stored!" - else - log_info "#{message.from} No data!" - end - imap.store(message_id, '+FLAGS', [:Deleted]) - log_info "#{message.from} Email processed!" + + # Imap logout and disconnection + def logout + imap.close + imap.logout + imap.disconnect + rescue StandardError => e + log.error("Cannot logout #{server}", e.message) end - rescue StandardError => e - log_error 'Error on mailcollector handle_new_mail' - log_error e.backtrace.join('\n') - raise - end - def handle_new_message(message, helper) - dataset = helper.prepare_new_dataset(message.subject) - message.attachments.each do |attachment| - tempfile = Tempfile.new('mail_attachment') - tempfile.binmode - tempfile.write(attachment.decoded) - tempfile.rewind - att = Attachment.new( - filename: attachment.filename, - created_by: helper.sender.id, - created_for: helper.recipient.id, - file_path: tempfile.path, - ) - - att.save! - tempfile.close - tempfile.unlink - att.update!(attachable: dataset) + # Set the IMAP instance + # + # @return [Net::IMAP] + def imap + @imap ||= Net::IMAP.new(server, port: port, ssl: ssl) end - rescue StandardError => e - log_error 'Error on mailcollector handle_new_message:' - log_error e.backtrace.join('\n') - raise - end - def email_eln_email?(mail_to) - mail_to.casecmp(@mail_address).zero? || - (@aliases.any? { |s| s.casecmp(mail_to).zero? }) - end + ############################################################################################################ + ## Building the Correspondence Array + ############################################################################################################ - def get_user(mail) - user = User.find_by email: mail - user ||= User.find_by email: mail.downcase - user - end + # Define the correspondence array between the ELN devices or users + # as extracted from the envelope (from, to, cc) + # + # @return [Array] the correspondence array + def correspondences + @correspondences ||= CorrespondenceArray.new(sender_email, receiver_emails) + end - def create_helper_set(envelope) - # Check if from is User or Device - from = "#{envelope.from[0].mailbox}@#{envelope.from[0].host}" - from_user = get_user from - raise "#{from} not registered" if from_user.nil? - - receiver = [] - if from_user.is_a?(User) && (!from_user.is_a?(Device) && !from_user.is_a?(Admin)) - receiver.push(from_user) - else # Concatenate all receiver (cc & to) - receiver.concat(envelope.cc) if envelope.cc - receiver.concat(envelope.to) if envelope.to - receiver = receiver.map { |m| "#{m.mailbox}@#{m.host}" } - .reject { |m| email_eln_email?(m) } - .map { |m| get_user(m) } - .select { |user| !user.nil? && !user.is_a?(Device) && !user.is_a?(Admin) } + # Check if the the given email address is the ELN system email address or one of the aliases + # + # @param mail_to [String] the email address to check + # @return [Boolean] true if the email address is the ELN system email address or one of the aliases + def email_eln_email?(mail_to) + mail_to.casecmp(mail_address).zero? || + (aliases.any? { |s| s.casecmp(mail_to).zero? }) end - return nil if receiver.empty? - CollectorHelperSet.new(from_user, receiver) - rescue StandardError => e - log_error 'Error on mailcollector create_helper:' - log_error e.backtrace.join('\n') - raise - end + # The sender email address from the current envelope + # + # @return [String] the email address of the sender + def sender_email + @envelope&.from&.first && format_email(@envelope.from.first) + end - def log_info(message) - DCLogger.log.info(self.class.name) do - " >>> #{message}" + # Format the email address from the email object + # + # @param email_object [Mail::Address] the email Object + # @return [String] the email address + def format_email(email_object) + "#{email_object.mailbox}@#{email_object.host}" + end + + # Extract the email address of the receivers from the envelope + # while omitting the ELN system email address + # + # @return [Array] the email addresses of the receivers + def receiver_emails + (@envelope.to.presence || []).concat(@envelope.cc || []) + .map { |email_obj| format_email(email_obj) } + .reject { |email| email_eln_email?(email) } + end + + ############################################################################################################ + ## Handling the Email + ############################################################################################################ + + # Set the envelope instance variable + # + # @param id [Integer] the id of the email to open + # @return [Net::IMAP::Envelope] the envelope of the email + def open_envelope(id) + @envelope = nil + @envelope = imap.fetch(id, ENVELOPE).first.attr[ENVELOPE] end - end - def log_error(message) - DCLogger.log.error(self.class.name) do - " >>> #{message}" + # Fetch the raw message from the email id + # + # @param id [Integer] the id of the email to Fetch + # @return [String] the raw message + def raw_message(id) + imap.fetch(id, RFC822).first.attr[RFC822] + end + + # Build the message object from the raw message and handle the attachments + # + # @param id [Integer] the id of the email to handle + def handle_message(id) + message = Mail.read_from_string raw_message(id) + return if message&.attachments.blank? + + log.info(message.from, message.subject) + message.attachments.each(&:handle_attachment) + rescue StandardError => e + log.error __method__, e.backtrace.join('\n') + ensure + reset_envelope + end + + # Attach the attachments of the email message to the receiver Inbox + # + # @param attachment [Mail::Attachment] the attachment to handle + def handle_attachment(attachment) + tempfile = Tempfile.new('mail_attachment') + tempfile.binmode + tempfile.write(attachment.decoded) + tempfile.rewind + correspondences.each do |correspondence| + correspondence.attach(attachment.filename, tempfile.path) + log.info("Attached #{attachment.filename} to #{correspondence.recipient}") + tempfile.rewind + end + rescue StandardError => e + log.error __method__, e.message + raise e + ensure + tempfile.close + tempfile.unlink end end end diff --git a/spec/api/chemotion/admin_device_api_spec.rb b/spec/api/chemotion/admin_device_api_spec.rb index 2c6c394eb6..9cc402ef07 100644 --- a/spec/api/chemotion/admin_device_api_spec.rb +++ b/spec/api/chemotion/admin_device_api_spec.rb @@ -27,7 +27,8 @@ context 'with params by name' do it 'fetches max 5 devices by name' do - get '/api/v1/admin_devices/byname?name=one&limit=5' + queried_name = URI::DEFAULT_PARSER.escape device.name[-3..].downcase + get "/api/v1/admin_devices/byname?name=#{queried_name}&limit=5" expect(parsed_json_response['devices'].size).to be(1) end end @@ -73,15 +74,15 @@ end it 'returns a valid connection' do - allow(Net::SFTP).to receive(:start).with( - device_with_sftp.datacollector_host, - device_with_sftp.datacollector_user, - key_data: [], - keys: Pathname.new(device_with_sftp.datacollector_key_name), - keys_only: true, - non_interactive: true, - timeout: 5, - ).and_return(sftp_double) + # allow(Net::SFTP).to receive(:start).with( + # device_with_sftp.datacollector_host, + # device_with_sftp.datacollector_user, + # key_data: [], + # keys: Pathname.new(device_with_sftp.datacollector_key_name), + # keys_only: true, + # non_interactive: true, + # timeout: 5, + # ).and_return(sftp_double) post '/api/v1/admin_devices/test_sftp', params: params expect(parsed_json_response['status']).to include('success') diff --git a/spec/factories/collector_datafiles.rb b/spec/factories/collector_datafiles.rb new file mode 100644 index 0000000000..a37608861d --- /dev/null +++ b/spec/factories/collector_datafiles.rb @@ -0,0 +1,218 @@ +# frozen_string_literal: true + +FactoryBot.define do + # Build a Pathname object for a file + # optionally create the file with touch or cp from another location + # @param [String] prefix - prefix for the filename + # @param [String] ext - extension for the file + # @param [Pathname] root - root directory for the file + # @param [Boolean] touch - create the file with touch + # @param [Pathname] copy_from - copy the file from another location + # @param [String] mode - file permissions + # @example + # build(:data_file, prefix: 'test', ext: 'txt', root: Pathname.new('/tmp')) + # # => # + factory :data_file, class: Pathname do + transient do + prefix { nil } + ext { 'chemotion' } + name { File.basename(Faker::File.file_name(ext: ext)) } + root { nil } + touch { true } # rubocop:disable Rails/SkipsModelValidations + copy_from { nil } + mode { 0o600 } + end + + initialize_with do + pathname = Pathname.new(prefix ? "#{prefix}-#{name}" : name) + pathname = root.join(pathname) if root + FileUtils.touch(pathname) if touch # rubocop:disable Rails/SkipsModelValidations + FileUtils.cp(copy_from, pathname) if copy_from + pathname.chmod(mode) if touch || copy_from # rubocop:disable Rails/SkipsModelValidations + pathname + end + end + + # Build a Pathname object for a folders, optionally: + # - create the folder with mkdir + # - populate the folder with dummy files + # @param [String] prefix - prefix for the folder name + # @param [String] name - name of the folder + # @param [Pathname] root - root directory for the folder + # @param [Boolean] mkdir - create the folder with mkdir_p + # @param [Integer] mode - folder permissions + # @param [Integer] file_count - number of dummy files to create + # @param [Integer] segment_count - number of additional segments in the folder path + factory :data_folder, class: Pathname do + transient do + prefix { nil } + name { Faker::File.dir(segment_count: 1) } + root { nil } + segment_count { 0 } + mkdir { true } + mode { 0o700 } + file_mode { 0o600 } + file_count { 0 } + end + + trait :with_files do + file_count { 2 } + end + + initialize_with do + pathname = Pathname.new(Faker::File.dir(root: root, segment_count: segment_count)) + .join(prefix ? "#{prefix}-#{name}" : name.to_s) + FileUtils.mkdir_p(pathname, mode: mode) if mkdir + FileUtils.chmod(mode, pathname) if mkdir + build_list(:data_file, file_count, root: pathname, touch: mkdir, mode: file_mode) + + pathname + end + end + + # Build a Pathname object for a parent folder + # populate the folder with folders [DataFolder] + # @return [Pathname] - the parent folder only + factory :data_folder_parent, parent: :data_folder do + transient do + prefixes { [] } + data_count { 1 } + end + initialize_with do + pathname = build(:data_folder, prefix: prefix, name: name, root: root, mode: mode) + prefixes.map do |prefix| + build_list( + :data_folder, + data_count, + root: pathname, + prefix: prefix, + file_count: file_count, + mode: mode, + file_mode: file_mode, + ) + end + pathname + end + end + + # Build a Pathname object for a parent folder + # populate the folder with files [DataFile] + # @return [Pathname] - the parent folder + factory :data_file_parent, parent: :data_folder_parent do + initialize_with do + pathname = build(:data_folder, prefix: prefix, name: name, root: root, mode: mode) + prefixes.map do |prefix| + build_list( + :data_file, + data_count, + root: pathname, + prefix: prefix, + mode: file_mode, + ) + end + pathname + end + end + + factory :data_for_folder_collector, parent: :data_folder_parent do + transient do + device { nil } + user_identifiers { [] } + name { '' } + data_count { 1 } + file_count { 1 } + prefixes { user_identifiers } + root { device&.datacollector_dir } + end + end + + factory :data_for_folder_collector_with_user_level, parent: :data_for_folder_collector do + initialize_with do + pathname = build(:data_folder, prefix: prefix, name: name, root: root, mode: mode) + user_identifiers.map do |username| + user_path = build(:data_folder, name: username, root: pathname, mode: mode) + build_list( + :data_folder, + data_count, + root: user_path, + file_count: file_count, + mode: mode, + file_mode: file_mode, + ) + end + pathname + end + end + + factory :data_for_file_collector, parent: :data_file_parent do + transient do + device { nil } + user_identifiers { [] } + name { '' } + data_count { 1 } + prefixes { user_identifiers } + root { device.datacollector_dir } + end + end + + factory :data_for_file_collector_with_user_level, parent: :data_for_file_collector do + initialize_with do + pathname = build(:data_folder, prefix: prefix, name: name, root: root, mode: mode) + user_identifiers.map do |username| + user_path = build(:data_folder, name: username, root: pathname, mode: mode) + build_list( + :data_file, + data_count, + root: user_path, + mode: file_mode, + ) + end + pathname + end + end + + factory :data_for_collector, class: Pathname do + transient do + device { nil } + user_identifiers { [] } + data_count { 1 } + file_count { 1 } + end + initialize_with do + params = { + device: device, + user_identifiers: user_identifiers, + data_count: data_count, + file_count: file_count, + } + sftp_params = { + **params, + mode: 0o777, + file_mode: 0o666, + } + + case "#{device.datacollector_method}#{device.datacollector_user_level_selected ? 'user' : ''}" + when 'folderwatcherlocal' + build(:data_for_folder_collector, **params) + when 'folderwatcherlocaluser' + build(:data_for_folder_collector_with_user_level, **params) + when 'folderwatchersftp' + build(:data_for_folder_collector, **sftp_params) + when 'folderwatchersftpuser' + build(:data_for_folder_collector_with_user_level, **sftp_params) + when 'filewatcherlocal' + params.delete(:file_count) + build(:data_for_file_collector, **params) + when 'filewatcherlocaluser' + params.delete(:file_count) + build(:data_for_file_collector_with_user_level, **params) + when 'filewatchersftp' + sftp_params.delete(:file_count) + build(:data_for_file_collector, **sftp_params) + when 'filewatchersftpuser' + sftp_params.delete(:file_count) + build(:data_for_file_collector_with_user_level, **sftp_params) + end + end + end +end diff --git a/spec/factories/devices.rb b/spec/factories/devices.rb index 2f39970083..b354c55467 100644 --- a/spec/factories/devices.rb +++ b/spec/factories/devices.rb @@ -1,88 +1,91 @@ # frozen_string_literal: true +DC_SFTP_USER = ENV['DATACOLLECTOR_FACTORY_SFTP_USER'].presence || 'testuser' +DC_DIR = ENV['DATACOLLECTOR_FACTORY_DIR'].presence || + Rails.configuration.datacollectors.dig(:localcollectors, 0, :path) +DC_SFTP_KEY = ENV['DATACOLLECTOR_FACTORY_SFTP_KEY'].presence || 'id_test' +DC_SFTP_HOST = ENV['DATACOLLECTOR_FACTORY_SFTP_HOST'].presence || '127.0.0.1' + FactoryBot.define do factory :device do - sequence(:email) { |n| "device#{n}@foo.bar" } + transient do + start_sequence { 0 } + # @note: start_sequence cannot be used as the 2nd argument for the initial value of the sequence + # because it is not available at the time of the factory definition + sequence(:fake_id) { |n| n + start_sequence } + end + email { "device#{fake_id}@foo.bar" } first_name { 'Device' } - last_name { 'One' } - name { 'Device One' } + last_name { fake_id } + name { "#{first_name} #{last_name}" } sequence(:name_abbreviation) { "D#{SecureRandom.alphanumeric(3)}" } - trait :file_local do - datacollector_fields { true } - datacollector_method { 'filewatcherlocal' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } - datacollector_authentication { 'password' } - datacollector_number_of_files { 1 } - datacollector_user_level_selected { false } + # passthrough parameters for data factory + transient do + # user_identifiers: array of user identifiers to be used for generating folders and files + user_identifiers { [] } + # data_count: number of data folders or data files to be generated per user identifier + data_count { 1 } + # file_count: number of files or folder to be generated per user identifier + file_count { 1 } end - trait :file_sftp do + trait :collector do datacollector_fields { true } - datacollector_method { 'filewatchersftp' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } - datacollector_user { ENV['DATACOLLECTOR_TEST_USER'].presence || 'testuser' } - datacollector_host { '127.0.0.1' } - datacollector_authentication { 'keyfile' } - datacollector_key_name { "#{Dir.home}/.ssh/id_test" } + datacollector_dir do + File.join( + DC_DIR, + "#{name_abbreviation}-#{datacollector_method}#{datacollector_user_level_selected ? '-user' : ''}", + ).to_s + end datacollector_number_of_files { 1 } datacollector_user_level_selected { false } - end - - trait :file_sftp_password do - datacollector_fields { true } - datacollector_method { 'filewatchersftp' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } - datacollector_user { ENV['DATACOLLECTOR_TEST_USER'].presence || 'user1' } - datacollector_host { '127.0.0.1' } datacollector_authentication { 'password' } - datacollector_number_of_files { 1 } - datacollector_user_level_selected { false } end - trait :file_sftp_faulty do - datacollector_fields { true } - datacollector_method { 'filewatchersftp' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } - datacollector_user { 'dummy' } - datacollector_host { '127.0.0.1' } + trait :sftp_collector do + collector + datacollector_host { DC_SFTP_HOST } + datacollector_user { DC_SFTP_USER } datacollector_authentication { 'keyfile' } - datacollector_key_name { "#{Dir.home}/.ssh/id_test" } - datacollector_number_of_files { 1 } - datacollector_user_level_selected { false } + datacollector_key_name { DC_SFTP_KEY } + end + + trait :file_local do + datacollector_method { 'filewatcherlocal' } + collector end trait :folder_local do - datacollector_fields { true } datacollector_method { 'folderwatcherlocal' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } - datacollector_authentication { 'password' } - datacollector_number_of_files { 1 } - datacollector_user_level_selected { false } + collector + end + + trait :file_sftp do + datacollector_method { 'filewatchersftp' } + sftp_collector end trait :folder_sftp do - datacollector_fields { true } datacollector_method { 'folderwatchersftp' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } - datacollector_user { ENV['DATACOLLECTOR_TEST_USER'].presence || 'testuser' } - datacollector_host { '127.0.0.1' } - datacollector_authentication { 'keyfile' } - datacollector_key_name { "#{Dir.home}/.ssh/id_test" } - datacollector_number_of_files { 1 } - datacollector_user_level_selected { false } + sftp_collector + end + + trait :file_sftp_password do + sftp_collector + datacollector_authentication { 'password' } + datacollector_key_name { nil } + end + + trait :file_sftp_faulty do + file_sftp + datacollector_user { 'dummy' } end trait :folder_sftp_faulty do - datacollector_fields { true } - datacollector_method { 'folderwatchersftp' } - datacollector_dir { Rails.root.join('tmp', 'datacollector', name_abbreviation) } + folder_sftp datacollector_user { 'dummy' } - datacollector_host { '127.0.0.1' } - datacollector_authentication { 'keyfile' } - datacollector_key_name { "#{Dir.home}/.ssh/id_test" } - datacollector_number_of_files { 1 } - datacollector_user_level_selected { false } + datacollector_host { '127.0.0.1:443' } end trait :novnc_settings do @@ -90,28 +93,20 @@ novnc_token { 'test' } end - before(:create) do |device| - keyfile = device.datacollector_key_name - - if keyfile.present? - dir = Pathname.new(Dir.home).join('.ssh') - FileUtils.mkdir_p(dir) unless File.directory?(dir) - FileUtils.cp(Rails.root.join('spec/fixtures/datacollector/id_test'), dir) - end - end - - before(:create) do |device| - destination = device.datacollector_dir - - if destination.present? - dir = if device.datacollector_method.include? 'folder' - Pathname.new(destination).join("CU1-#{Time.now.to_i}") - else - destination - end - - FileUtils.mkdir_p(dir) unless File.directory?(dir) - FileUtils.cp(Rails.root.join('spec/fixtures/CU1-folder/CU1-abc.txt'), dir) + before(:create) do |device, evaluator| + # create datacollector_dir if it is set + if device.datacollector_dir.present? + build(:data_folder, root: DC_DIR, name: '', mode: 0o755) + build(:data_folder, root: device.datacollector_dir, name: '', mode: 0o755) + if evaluator.user_identifiers.present? + build( + :data_for_collector, + device: device, + user_identifiers: evaluator.user_identifiers, + data_count: evaluator.data_count, + file_count: evaluator.file_count, + ) + end end end end diff --git a/spec/fixtures/datacollector/CU1-test.txt b/spec/fixtures/datacollector/CU1-test.txt new file mode 100644 index 0000000000..345e6aef71 --- /dev/null +++ b/spec/fixtures/datacollector/CU1-test.txt @@ -0,0 +1 @@ +Test diff --git a/spec/lib/datacollector/collector_file_spec.rb b/spec/lib/datacollector/collector_file_spec.rb new file mode 100644 index 0000000000..6e6e1d9585 --- /dev/null +++ b/spec/lib/datacollector/collector_file_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Datacollector::CollectorFile do + let(:root_path) do + build( + :data_folder, + root: Rails.configuration.datacollectors.dig(:localcollectors, 0, :path), + name: '', + mode: 0o755, + mkdir: true, + ) + end + let(:relative_path) { 'test' } + let(:full_path) do + build(:data_file, root: root_path, name: relative_path, touch: true) + end + + describe '#initialize' do + context 'when the root path is invalid' do + it 'raises an ArgumentError' do + expect { described_class.new!(relative_path, '/invalid/root/path') }.to raise_error(ArgumentError) + end + end + + context 'when the relative path is invalid' do + it 'raises an ArgumentError' do + expect { described_class.new!('../invalid/path', root_path) }.to raise_error(ArgumentError) + end + end + + context 'when the paths are valid' do + it 'initializes the object correctly' do + full_path + fstruct = described_class.new!(relative_path, root_path) + expect(fstruct.root_path).to eq(root_path.to_s) + expect(fstruct.relative_path).to eq(relative_path.to_s) + expect(fstruct.path).to eq(full_path.to_s) + full_path.delete + end + end + end + + describe '#mtime' do + context 'when the file exists locally' do + it 'returns the modification time' do + full_path + fstruct = described_class.new(relative_path, root_path) + expect(fstruct.mtime).to be_a(Time) + full_path.delete + end + end + + context 'when the file exists on SFTP' do + let(:sftp) { instance_double(Net::SFTP::Session) } + let(:sftp_stat) { instance_double(Net::SFTP::Protocol::V01::Attributes, mtime: Time.now.to_i) } + + before do + allow(sftp).to receive(:stat!).with(full_path).and_return(sftp_stat) + end + + it 'returns the modification time' do + fstruct = described_class.new(relative_path, root_path, sftp: sftp) + expect(fstruct.mtime).to be_a(Time) + end + end + end + + describe '#delete' do + context 'when the file exists locally' do + before do + allow(File).to receive(:exist?).with(full_path).and_return(true) + allow(FileUtils).to receive(:rm).with(full_path) + end + + it 'deletes the file' do + fstruct = described_class.new(relative_path, root_path) + expect(fstruct.delete).to be true + end + end + + context 'when the file exists on SFTP' do + let(:sftp) { instance_double(Net::SFTP::Session) } + + before do + allow(sftp).to receive(:remove!).with(full_path) + end + + it 'deletes the file' do + fstruct = described_class.new(relative_path, root_path, sftp: sftp) + expect(fstruct.delete).to be true + end + end + end +end diff --git a/spec/lib/datacollector/collector_spec.rb b/spec/lib/datacollector/collector_spec.rb new file mode 100644 index 0000000000..0dbde22144 --- /dev/null +++ b/spec/lib/datacollector/collector_spec.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Datacollector::Collector, type: :model do + let(:users) { create_list(:person, 2) } + let(:users_unknown) { build_list(:person, 1) } + let(:name_abbrs) { (users + users_unknown).map(&:name_abbreviation) } + let(:emails) { (users + users_unknown).map(&:email) } + let(:data_count) { 1 } + let(:file_count) { 1 } + + describe '.execute' do + # @todo context sftp-keyfile-authentication and sftp-password-authentication + # @todo context undeletable files and bypassing them / registering collector errors + # @todo context files not ready (creation time < sleep time) + + shared_examples 'a working collector without user directories' do + it 'executes and writes the correct number of files in database' do + collector = described_class.new(device) + raise 'example not ready ' unless collector_data.to_s == collector.collector_dir + + expect(Dir).not_to be_empty(collector.collector_dir) + expect { collector.execute }.to change(Attachment, :count).by(data_count * users.count) + expect(Dir).to be_empty(collector.collector_dir) + end + end + + shared_examples 'a working collector with user directories' do + it 'executes and writes the correct number of files in database' do + collector = described_class.new(device) + path = collector_data.join(users.first.name_abbreviation) + path_unknown = collector_data.join(users_unknown.first.name_abbreviation) + + expect(Dir.empty?(path) || Dir.empty?(path_unknown)).to be_falsey + expect { collector.execute }.to change(Attachment, :count).by(data_count * users.count) + # real user dir to be empty, unknown user dir to be untouched + expect(!Dir.empty?(path) || Dir.empty?(path_unknown)).to be_falsey + end + end + + user_level_options = [true, false] + collector_options = { + file_local: 'local file', + folder_local: 'local folder', + file_sftp: 'sftp file', + folder_sftp: 'sftp folder', + } + collector_options.each do |device_trait, description| + user_level_options.each do |user_level| + context "with a device configured for #{description} #{user_level ? '(user dirs)' : ''}" do + let(:device) do + create( + :device, + device_trait, + datacollector_user_level_selected: user_level, + datacollector_number_of_files: file_count, + ) + end + let(:collector_data) do + build( + :data_for_collector, + device: device, + user_identifiers: name_abbrs, + data_count: data_count, + file_count: file_count, + ) + end + + example = "a working collector with#{user_level ? '' : 'out'} user directories" + it_behaves_like example + end + end + end + + context 'when folder is not ready for collection' do + # let(:file_count) { 2 } + let(:device) do + create( + :device, + :folder_local, + datacollector_number_of_files: file_count + 1, + ) + end + + it 'does not process the folder when not enough file are there' do + device_dir = build( + :data_for_collector, device: device, user_identifiers: name_abbrs[0..0], file_count: file_count + ) + collector = described_class.new(device) + expected_files = device.datacollector_number_of_files.to_i - 1 + # Ensure test files are set up and test execution + # rubocop:disable Performance/Count + expect(device_dir.glob('**/*').select(&:file?).count).to eq(expected_files) + expect { collector.execute }.not_to change(Attachment, :count) + expect(device_dir.glob('**/*').select(&:file?).count).to eq(expected_files) + # rubocop:enable Performance/Count + end + + it 'does not process the folder when the file has just been created' do + expected_files = device.datacollector_number_of_files.to_i + device_dir = build( + :data_for_collector, device: device, user_identifiers: name_abbrs[0..0], file_count: expected_files + ) + collector = described_class.new(device) + allow(collector.config).to receive(:sleep_time).and_return(10) + # Ensure test files are set up and test execution + # rubocop:disable Performance/Count + expect(device_dir.glob('**/*').select(&:file?).count).to eq(expected_files) + expect { collector.execute }.not_to change(Attachment, :count) + expect(device_dir.glob('**/*').select(&:file?).count).to eq(expected_files) + # rubocop:enable Performance/Count + end + + it 'does not process the folder when no files are there' do + device_dir = build(:data_for_collector, device: device, user_identifiers: name_abbrs[0..0], file_count: 0) + collector = described_class.new(device) + # Ensure test files are set up and test execution + # rubocop:disable Performance/Count + expect(device_dir.glob('**/*').select(&:file?).count).to eq(0) + expect { collector.execute }.not_to change(Attachment, :count) + expect(device_dir.glob('**/*').select(&:directory?).count).to eq(1) + # rubocop:enable Performance/Count + end + end + end + + describe '#bulk_execute' do + context 'when a device has non-working sftp config' do + let(:devices) do + [ + create(:device, :folder_sftp_faulty, user_identifiers: name_abbrs[0..0], data_count: 2 * data_count), + create( + :device, + :file_sftp_faulty, + datacollector_user_level_selected: true, + user_identifiers: name_abbrs[0..0], + data_count: 2 * data_count, + ), + create(:device, :folder_local, user_identifiers: name_abbrs[0..0], data_count: data_count), + ] + end + + it 'bypasses faulty devices' do + expect do + described_class.bulk_execute(devices) + end.to change(Attachment, :count).by(data_count) + end + end + end +end diff --git a/spec/lib/datacollector/correspondence_helpers_spec.rb b/spec/lib/datacollector/correspondence_helpers_spec.rb new file mode 100644 index 0000000000..2ef43821ce --- /dev/null +++ b/spec/lib/datacollector/correspondence_helpers_spec.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Datacollector::CorrespondenceHelpers do + # set some users and devices for the tests + # + let(:user_person) { create(:person) } + let(:user_group) { create(:group) } + let(:user_admin) { create(:admin) } + let(:device) { create(:device) } + let(:device_eponyme) { create(:device, name_abbreviation: user_person.name_abbreviation) } + + # raw identifiers are made from an email addresses or a name_abbreviation: + # - user names can be used as prefix to filenames + # - name_abbreviations and emails addresses can be used as directory names + + let(:person_email_as_dir) do + Pathname.new(Faker::File.dir(root: '/tmp', segment_count: 3)).join(user_person.email).to_s + end + let(:person_name_abbreviation_as_dir) do + Pathname.new(Faker::File.dir(root: '/tmp', segment_count: 3)).join(user_person.name_abbreviation).to_s + end + let(:person_name_abbreviation_as_file_prefix) do + "#{user_person.name_abbreviation}-#{File.basename(Faker::File.file_name(dir: ''))}" + end + + let(:correspondence) { Class.new { extend Datacollector::CorrespondenceHelpers } } + + shared_examples 'find_user' do + it 'finds the user' do + expect(correspondence.find_user(identifier)).to eq(user) + end + end + + describe '.parse_identifier' do + it 'extracts the name_abbreviation or email from a dir path or file name prefix' do + expect( + correspondence.send(:parse_identifier, person_email_as_dir), + ).to eq(user_person.email) + expect( + correspondence.send(:parse_identifier, person_name_abbreviation_as_dir), + ).to eq(user_person.name_abbreviation) + expect( + correspondence.send(:parse_identifier, person_name_abbreviation_as_file_prefix), + ).to eq(user_person.name_abbreviation) + end + + it 'returns the identifier when only the identifier is given' do + %i[name_abbreviation email].each do |attribute| + [user_person, user_group, user_admin, device].each do |subject| + identifier = subject.send(attribute) + expect(correspondence.send(:parse_identifier, identifier)).to eq(identifier) + end + end + end + end + + describe '.find_user' do + %i[name_abbreviation email].each do |attribute| + it "finds the user per #{attribute}" do + [user_person, user_group].each do |subject| + identifier = subject.send(attribute) + expect(correspondence.find_user(identifier)).to eq(subject) + end + end + + it "does not find an admin per #{attribute}" do + [user_admin].each do |user| + identifier = user.send(attribute) + expect(correspondence.find_user(identifier)).to be_nil + end + end + end + + it 'does not find a user with a wrong name abbreviation' do + false_name_abbreviation_long = user_person.name_abbreviation + Faker::Alphanumeric.alpha(number: 1) + false_name_abbreviation_short = user_person.name_abbreviation[0..-2] + [false_name_abbreviation_long, false_name_abbreviation_short].each do |false_name_abbreviation| + expect(correspondence.find_user(false_name_abbreviation)).to be_nil + end + end + end + + # @note atm, only email seems relevant to find devices as it is only used in mail collector + describe '.find_device' do + %i[name_abbreviation email].each do |attribute| + it "finds a Device per #{attribute}" do + expect(correspondence.find_device(device.name_abbreviation)).to eq(device) + end + end + end + + describe '.find_sender' do + context 'when a user and a device share the same name_abbreviation' do + it 'finds the device over the user' do + expect(user_person.name_abbreviation).to eq(device_eponyme.name_abbreviation) + expect(correspondence.find_sender(user_person.name_abbreviation)).to eq(device_eponyme) + end + end + end + + describe '.validate' do + it 'returns true if the object is an activated Person account or a device' do + [Person.new(account_active: true), Device.new].each do |object| + expect(correspondence.validate(object)).to be_truthy + end + end + + it 'returns false if the user is not activated or an admin' do + [Person.new(account_active: false), Admin.new(account_active: true)].each do |user| + expect(correspondence.validate(user)).to be_falsey + end + end + end +end diff --git a/spec/lib/datacollector/correspondence_spec.rb b/spec/lib/datacollector/correspondence_spec.rb new file mode 100644 index 0000000000..47f9c63d8d --- /dev/null +++ b/spec/lib/datacollector/correspondence_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Datacollector::Correspondence do + # Test the initialize method: + # it create a new correspondence object when + # - a device or user can be found in the database with the given id as from argument + # - a device or user are passed as from argument + # and + # - a user can be found in the database with the given id as to argument + # - a user is passed as to argument + + # set some users and devices for the tests + # + let(:person) { create(:person) } + let(:device) { create(:device, :file_local, user_identifiers: [person.name_abbreviation]) } + + describe described_class do + subject(:correspondence) { described_class.new(device, person) } + + # rubocop:disable RSpec/BeforeAfterAll + after(:all) do + Pathname.new(Rails.configuration.datacollectors.dig(:localcollectors, 0, :path)).rmtree + end + # rubocop:enable RSpec/BeforeAfterAll + + describe '.new' do + it 'returns a new instance of the class' do + expect(correspondence).to be_a(described_class) + end + + it 'has prepared the containers for the recipient' do + container = correspondence.sender_container + expect(container).to be_a(Container) + expect(person.container.descendants).to include(container) + end + end + + describe '#sender_container' do + it 'has the proper name and container_type' do + container = correspondence.sender_container + expect(container.name).to eq(device.name) + expect(container.container_type).to eq(correspondence.send(:build_sender_box_id, device.id)) + end + end + + describe '#sender' do + it 'returns the sender' do + expect(correspondence.sender).to eq(device) + end + end + + describe '#recipient' do + it 'returns the recipient' do + expect(correspondence.recipient).to eq(person) + end + end + + describe 'attaching a file' do + subject(:correspondence) { described_class.new(device, person) } + + it 'attaches a file to the correspondence' do + file = Pathname.new(device.datacollector_dir).glob('*').first + name = file.basename.to_s + correspondence.attach(name, file.to_s) + attachment = correspondence.sender_container.descendants.filter_map(&:attachments).flatten.first + + expect(attachment.filename).to eq(name) + end + end + end +end diff --git a/spec/lib/datacollector/datacollector_file_spec.rb b/spec/lib/datacollector/datacollector_file_spec.rb deleted file mode 100644 index f76a6d6e4e..0000000000 --- a/spec/lib/datacollector/datacollector_file_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe DatacollectorFile, type: :model do - let(:user) { create(:person, name_abbreviation: 'CU1') } - let(:device) { create(:device, users: [user]) } - let(:expected_attachment) { Attachment.find_by(filename: 'CU1-abc.txt') } - - describe '.collect_from' do - context 'when have valid file' do - before do - described_class.new(Rails.root.join('spec/fixtures/CU1-folder/CU1-abc.txt').to_s, nil).collect_from(device) - end - - it 'attachment is saved' do - expect(expected_attachment).not_to be_nil - end - - it 'filedata was attached' do - expect(expected_attachment.attachment_data).not_to be_nil - end - end - end -end diff --git a/spec/lib/datacollector/datacollector_folder_spec.rb b/spec/lib/datacollector/datacollector_folder_spec.rb deleted file mode 100644 index 0401876d70..0000000000 --- a/spec/lib/datacollector/datacollector_folder_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe DatacollectorFolder, type: :model do - let(:user) { create(:person, name_abbreviation: 'CU1') } - let(:device) { create(:device, users: [user]) } - let(:attachment) { Attachment.find_by(filename: 'CU1-folder.zip') } - - describe '.collect_from' do - context 'when have valid file' do - before do - datacollector = described_class.new(Rails.root.join('spec/fixtures/CU1-folder').to_s, nil) - datacollector.files = [File.join('CU1-abc.txt')] - datacollector.collect(device) - end - - it 'attachment was saved' do - expect(attachment).not_to be_nil - end - - it 'file was correctly attached' do - expect(attachment.attachment_data).not_to be_nil - end - end - end -end diff --git a/spec/lib/datacollector/filecollector_spec.rb b/spec/lib/datacollector/filecollector_spec.rb deleted file mode 100644 index e2240086f5..0000000000 --- a/spec/lib/datacollector/filecollector_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Filecollector, type: :model do - let(:user) { create(:person, name_abbreviation: 'CU1') } - let(:device1) { create(:device, :file_local, users: [user]) } - let(:device2) { create(:device, :file_local, users: [user]) } - let(:device_sftp1) { create(:device, :file_sftp, users: [user]) } - let(:device_sftp2) { create(:device, :file_sftp_faulty, users: [user]) } - let(:device_sftp3) { create(:device, :file_sftp, users: [user]) } - - describe '.execute' do - context 'when files are collected without error over local connection' do - it 'executes and writes the correct number of files in database' do - device1 - device2 - - expect { described_class.new.execute(false) }.to change(Attachment, :count).by(2) - end - end - - context 'when files are collected without error over sftp connection' do - it 'executes and writes the correct number of files in database' do - device_sftp1 - device_sftp3 - - expect { described_class.new.execute(true) }.to change(Attachment, :count).by(2) - end - end - - context 'when devices connect with keyfile' do - it 'connects and writes the correct number of files in database' do - device_sftp1 - device_sftp3 - - expect { described_class.new.execute(true) }.to change(Attachment, :count).by(2) - end - end - - context 'when there is authentication error' do - it 'bypasses faulty device and move to next one' do - device_sftp1 - device_sftp2 - device_sftp3 - - expect { described_class.new.execute(true) }.to change(Attachment, :count).by(2) - end - end - end -end diff --git a/spec/lib/datacollector/foldercollector_spec.rb b/spec/lib/datacollector/foldercollector_spec.rb deleted file mode 100644 index 02546ba626..0000000000 --- a/spec/lib/datacollector/foldercollector_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Foldercollector, type: :model do - let(:user) { create(:person, name_abbreviation: 'CU1') } - let(:device1) { create(:device, :folder_local, users: [user]) } - let(:device2) { create(:device, :folder_local, users: [user]) } - let(:device_sftp1) { create(:device, :folder_sftp, users: [user]) } - let(:device_sftp2) { create(:device, :folder_sftp_faulty, users: [user]) } - let(:device_sftp3) { create(:device, :folder_sftp, users: [user]) } - - describe '.execute' do - context 'when files are collected without error over local connection' do - it 'executes and writes the correct number of files in database' do - device1 - device2 - expect { described_class.new.execute(false) }.to change(Attachment, :count).by(Device.count) - end - end - - context 'when files are collected without error over sftp connection' do - it 'executes and writes the correct number of files in database' do - device_sftp1 - device_sftp3 - - expect { described_class.new.execute(true) }.to change(Attachment, :count).by(Device.count) - end - end - - context 'when devices connect with keyfile' do - it 'connects and writes the correct number of files in database' do - device_sftp1 - device_sftp3 - - expect { described_class.new.execute(true) }.to change(Attachment, :count).by(Device.count) - end - end - - context 'when there is authentication error' do - it 'bypasses faulty device and move to next one' do - device_sftp1 - device_sftp2 - device_sftp3 - expect { described_class.new.execute(true) }.to change(Attachment, :count).by(2) - end - end - end -end diff --git a/spec/lib/datacollector/mailcollector_spec.rb b/spec/lib/datacollector/mailcollector_spec.rb new file mode 100644 index 0000000000..66f7956f3c --- /dev/null +++ b/spec/lib/datacollector/mailcollector_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Datacollector::Mailcollector do + let(:users) { create_list(:person, 2) } + let(:users_unknown) { build_list(:person, 1) } + let(:name_abbrs) { (users + users_unknown).map(&:name_abbreviation) } + let(:emails) { (users + users_unknown).map(&:email) } + let(:mail_collector) { described_class.new } + + describe '.initialize' do + # rubocop:disable RSpec/AnyInstance + before do + allow_any_instance_of(described_class).to receive(:login).and_return(true) + end + # rubocop:enable RSpec/AnyInstance + + it 'initializes the object' do + expect(mail_collector).to be_instance_of(described_class) + expect(mail_collector.config).to be_instance_of(Datacollector::MailConfiguration) + end + + it 'returns the server address' do + expect(mail_collector.server).to eq(Rails.configuration.datacollectors[:mailcollector][:server]) + end + end +end