diff --git a/bin/run_mail_collector b/bin/run_mail_collector new file mode 100644 index 0000000000..82d2a6b161 --- /dev/null +++ b/bin/run_mail_collector @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) +CollectDataFromMailJob.perform_now + diff --git a/lib/datacollector/collector_helper.rb b/lib/datacollector/collector_helper.rb index c4a2a3e2d4..a60ba741aa 100644 --- a/lib/datacollector/collector_helper.rb +++ b/lib/datacollector/collector_helper.rb @@ -1,20 +1,22 @@ +# frozen_string_literal: true + class CollectorHelper attr_reader :sender, :sender_container, :recipient def initialize(from, cc = nil) - if from.is_a?(Device) && cc.is_a?(User) + if (from.is_a?(Device) && cc.is_a?(User)) || (from.is_a?(User) && from == cc) @sender = from @recipient = cc prepare_containers elsif cc @sender = Device.find_by email: from - @sender = Device.find_by email: from.downcase unless @sender + @sender ||= Device.find_by email: from.downcase @recipient = User.find_by email: cc - @recipient = User.find_by email: cc.downcase unless @recipient + @recipient ||= User.find_by email: cc.downcase prepare_containers if @sender && @recipient else @sender = User.find_by email: from - @sender = User.find_by email: from.downcase unless @sender + @sender ||= User.find_by email: from.downcase @recipient = @sender prepare_containers if @sender end @@ -26,19 +28,21 @@ def sender_recipient_known? def prepare_new_dataset(subject) return nil unless sender_recipient_known? + Container.create( name: subject, container_type: 'dataset', - parent: @sender_container + parent: @sender_container, ) end def prepare_dataset(subject) return nil unless sender_recipient_known? + Container.where( name: subject, container_type: 'dataset', - parent: @sender_container + parent: @sender_container, ).first_or_create end @@ -65,14 +69,14 @@ def prepare_containers unless @recipient.container @recipient.container = Container.create( name: 'inbox', - container_type: 'root' + container_type: 'root', ) end - sender_box_id = 'sender_box_' + @sender.id.to_s + sender_box_id = "sender_box_#{@sender.id}" @sender_container = Container.where( name: @sender.first_name, container_type: sender_box_id, - parent_id: @recipient.container.id + parent_id: @recipient.container.id, ).first_or_create end end diff --git a/lib/datacollector/collector_helper_set.rb b/lib/datacollector/collector_helper_set.rb new file mode 100644 index 0000000000..0ffefc25fe --- /dev/null +++ b/lib/datacollector/collector_helper_set.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class CollectorHelperSet + attr_reader :helper_set + + def initialize(from, cc_list) + @helper_set = [] + cc_list.each do |cc| + h = CollectorHelper.new(from, cc) + @helper_set.push(h) if h.sender_recipient_known? + end + end + + def sender_recipient_known? + @helper_set.length.positive? + end +end diff --git a/lib/datacollector/datacollector_file.rb b/lib/datacollector/datacollector_file.rb index fc9795a0c0..d42075ae0e 100644 --- a/lib/datacollector/datacollector_file.rb +++ b/lib/datacollector/datacollector_file.rb @@ -19,7 +19,7 @@ def delete def attach(device) att = Attachment.new( filename: @name, - file_data: IO.binread(@path), + file_data: File.binread(@path), content_type: MimeMagic.by_path(@name)&.type, created_by: device.id, created_for: recipient.id, @@ -52,7 +52,7 @@ def attach_remote(device) def add_attach_to_container(device, attach, _ = false) helper = CollectorHelper.new(device, recipient) - dataset = helper.prepare_dataset(Time.now.strftime('%Y-%m-%d')) + dataset = helper.prepare_dataset(Time.zone.now.strftime('%Y-%m-%d')) attach.update!(attachable: dataset) # add notifications diff --git a/lib/datacollector/datacollector_folder.rb b/lib/datacollector/datacollector_folder.rb index e9d973d2b4..bd45980918 100644 --- a/lib/datacollector/datacollector_folder.rb +++ b/lib/datacollector/datacollector_folder.rb @@ -47,7 +47,7 @@ def zip_files(tmpzip) def register_new_data(device, tmpzip) att = Attachment.new( - filename: @name + '.zip', + filename: "#{@name}.zip", created_by: device.id, created_for: recipient.id, content_type: 'application/zip', diff --git a/lib/datacollector/dc_logger.rb b/lib/datacollector/dc_logger.rb index b82ea9b02e..002546de9c 100644 --- a/lib/datacollector/dc_logger.rb +++ b/lib/datacollector/dc_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class DCLogger def self.log @@fw_logger ||= Logger.new("#{Rails.root}/log/datacollector.log") diff --git a/lib/datacollector/fcollector.rb b/lib/datacollector/fcollector.rb index 2c9c3ed671..1c774690b8 100644 --- a/lib/datacollector/fcollector.rb +++ b/lib/datacollector/fcollector.rb @@ -63,7 +63,7 @@ def execute(use_sftp) inspect_folder(device) @sftp = nil end - rescue => e + rescue StandardError => e log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") end end @@ -77,7 +77,7 @@ def execute(use_sftp) private def devices(use_sftp) - sql = <<~SQL + sql = <<~SQL.squish profiles."data"->>'method' = '#{self.class::FCOLL}watcher#{use_sftp ? 'sftp' : 'local'}' SQL Device.joins(:profile).where(sql).includes(:profile) diff --git a/lib/datacollector/filecollector.rb b/lib/datacollector/filecollector.rb index d64f41bbac..c9088530d9 100644 --- a/lib/datacollector/filecollector.rb +++ b/lib/datacollector/filecollector.rb @@ -7,7 +7,6 @@ class Filecollector < Fcollector private # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/MethodLength def inspect_folder(device) directory = device.profile.data['method_params']['dir'] @@ -15,7 +14,7 @@ def inspect_folder(device) @current_collector = DatacollectorFile.new(new_file_p, @sftp) error = CollectorError.find_by error_code: CollectorHelper.hash( @current_collector.path, - @sftp + @sftp, ) begin stored = false @@ -31,10 +30,10 @@ def inspect_folder(device) @current_collector.delete log_info("Recipient unknown. File deleted! >>> #{device.info}") end - rescue => e + rescue StandardError => e if stored CollectorHelper.write_error( - CollectorHelper.hash(@current_collector.path, @sftp) + CollectorHelper.hash(@current_collector.path, @sftp), ) end log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") @@ -53,15 +52,13 @@ def new_files(monitored_folder_p) File.join(monitored_folder_p, f.name) end else - new_files_p = Dir.glob(File.join(monitored_folder_p, '*')).reject { |e| + new_files_p = Dir.glob(File.join(monitored_folder_p, '*')).reject do |e| File.directory?(e) - } + end end new_files_p.delete_if do |f| f.end_with?('.filepart', '.part') end new_files_p end - - # rubocop:enable Metrics/MethodLength end diff --git a/lib/datacollector/foldercollector.rb b/lib/datacollector/foldercollector.rb index a9bcb1f35e..4c78cee117 100644 --- a/lib/datacollector/foldercollector.rb +++ b/lib/datacollector/foldercollector.rb @@ -9,22 +9,20 @@ class Foldercollector < Fcollector private def sleep_seconds(device) - 30 || Rails.configuration.datacollectors.services && - (Rails.configuration.datacollectors.services.find { |e| + 30 || (Rails.configuration.datacollectors.services && + (Rails.configuration.datacollectors.services.find do |e| e[:name] == device.profile.data['method'] - } || {})[:watcher_sleep] + end || {})[:watcher_sleep]) end def modification_time_diff(device, folder_p) - time_now = Time.now - time_diff = - case device.profile.data['method'] - when 'folderwatcherlocal' then time_now - File.mtime(folder_p) - when 'folderwatchersftp' then - time_now - (Time.at @sftp.file.open(folder_p).stat.attributes[:mtime]) - else 30 - end - time_diff + time_now = Time.zone.now + case device.profile.data['method'] + when 'folderwatcherlocal' then time_now - File.mtime(folder_p) + when 'folderwatchersftp' + time_now - (Time.zone.at @sftp.file.open(folder_p).stat.attributes[:mtime]) + else 30 + end end # rubocop:disable Metrics/CyclomaticComplexity @@ -45,7 +43,7 @@ def inspect_folder(device) @current_collector.files = list_files error = CollectorError.find_by error_code: CollectorHelper.hash( @current_collector.path, - @sftp + @sftp, ) begin stored = false @@ -66,10 +64,10 @@ def inspect_folder(device) @current_collector.delete log_info("Recipient unknown. Folder deleted! >>> #{device.info}") end - rescue => e + rescue StandardError => e if stored CollectorHelper.write_error( - CollectorHelper.hash(@current_collector.path, @sftp) + CollectorHelper.hash(@current_collector.path, @sftp), ) end log_error("#{e.message} >>> #{device.info}\n#{e.backtrace.join('\n')}") @@ -89,9 +87,9 @@ def list_files ) all_files.map!(&:name) else - all_files = Dir.entries(@current_collector.path).reject { |e| + all_files = Dir.entries(@current_collector.path).reject do |e| File.directory?(File.join(@current_collector.path, e)) - } + end end all_files.delete_if do |f| f.end_with?('..', '.', '.filepart', '.part') @@ -106,9 +104,9 @@ def new_folders(monitored_folder_p) ) new_folders_p.map! { |dir| File.join(monitored_folder_p, dir.name) } else - new_folders_p = Dir.glob(File.join(monitored_folder_p, '*')).select { |e| + new_folders_p = Dir.glob(File.join(monitored_folder_p, '*')).select do |e| File.directory?(e) - } + end end new_folders_p end diff --git a/lib/datacollector/mailcollector.rb b/lib/datacollector/mailcollector.rb index 25ae8d9d9b..b6933c06b2 100644 --- a/lib/datacollector/mailcollector.rb +++ b/lib/datacollector/mailcollector.rb @@ -1,8 +1,9 @@ +# frozen_string_literal: true + require 'net/imap' require 'mail' class Mailcollector - def initialize raise 'No datacollector configuration!' unless Rails.configuration.datacollectors @@ -21,15 +22,17 @@ def execute if response['name'] == 'OK' log_info('Login...') imap.select('INBOX') - imap.search(['NOT', 'SEEN']).each do |message_id| + imap.search(%w[NOT SEEN]).each do |message_id| handle_new_mail(message_id, imap) + rescue StandardError => e + log_error e.message end imap.close else - log_error('Cannot login ' + @server) + log_error("Cannot login #{@server}") raise end - rescue => e + rescue StandardError => e log_error 'mail collector execute error:' log_error e.backtrace.join('\n') raise @@ -44,117 +47,107 @@ def handle_new_mail(message_id, imap) envelope = imap.fetch(message_id, 'ENVELOPE')[0].attr['ENVELOPE'] raw_message = imap.fetch(message_id, 'RFC822').first.attr['RFC822'] message = Mail.read_from_string raw_message - helper = create_helper(envelope) - log_info 'Mail from ' + message.from.to_s - unless helper - log_info message.from.to_s + ' Email format incorrect or sender unknown!' - return nil - end - unless helper.sender - log_info message.from.to_s + ' Sender unknown!' + helper_set = create_helper_set(envelope) + log_info "Mail from #{message.from}" + unless helper_set + log_info "#{message.from} Email format incorrect or sender unknown!" return nil end - unless helper.recipient - log_info message.from.to_s + ' Recipient unknown!' - return nil - end - if message.attachments - handle_new_message(message, helper) - log_info message.from.to_s + ' Data stored!' - else - log_info message.from.to_s + ' No data!' + helper_set.helper_set.each do |helper| + unless helper.sender + log_info "#{message.from} Sender unknown!" + return nil + end + unless helper.recipient + log_info "#{message.from} Recipient unknown!" + return nil + end + if message.attachments + handle_new_message(message, helper) + log_info "#{message.from} Data stored!" + else + log_info "#{message.from} No data!" + end + imap.store(message_id, '+FLAGS', [:Deleted]) + log_info "#{message.from} Email processed!" end - imap.store(message_id, '+FLAGS', [:Deleted]) - log_info message.from.to_s + ' Email processed!' - rescue => e + rescue StandardError => e log_error 'Error on mailcollector handle_new_mail' log_error e.backtrace.join('\n') raise end def handle_new_message(message, helper) - begin - dataset = helper.prepare_new_dataset(message.subject) - message.attachments.each do |attachment| - tempfile = Tempfile.new('mail_attachment') - tempfile.binmode - tempfile.write(attachment.decoded) - tempfile.rewind - att = Attachment.new( - filename: attachment.filename, - created_by: helper.sender.id, - created_for: helper.recipient.id, - file_path: tempfile.path, - ) + dataset = helper.prepare_new_dataset(message.subject) + message.attachments.each do |attachment| + tempfile = Tempfile.new('mail_attachment') + tempfile.binmode + tempfile.write(attachment.decoded) + tempfile.rewind + att = Attachment.new( + filename: attachment.filename, + created_by: helper.sender.id, + created_for: helper.recipient.id, + file_path: tempfile.path, + ) - att.save! - tempfile.close - tempfile.unlink - att.update!(attachable: dataset) - end - rescue => e - log_error 'Error on mailcollector handle_new_message:' - log_error e.backtrace.join('\n') - raise + att.save! + tempfile.close + tempfile.unlink + att.update!(attachable: dataset) end + rescue StandardError => e + log_error 'Error on mailcollector handle_new_message:' + log_error e.backtrace.join('\n') + raise end - def create_helper(envelope) - helper = nil - begin - if (envelope.cc && envelope.cc.length == 1) && (envelope.to && envelope.to.length == 1) - log_info 'Using CC method...' - mail_to = envelope.to[0].mailbox.to_s + '@' + envelope.to[0].host.to_s - if mail_to.casecmp(@mail_address).zero? || - (@aliases.any? { |s| s.casecmp(mail_to).zero? }) - helper = CollectorHelper.new( - envelope.from[0].mailbox.to_s + '@' + envelope.from[0].host.to_s, - envelope.cc[0].mailbox.to_s + '@' + envelope.cc[0].host.to_s - ) - end - elsif !envelope.cc && envelope.to && envelope.to.length == 2 - log_info 'Using To method...' - mail_to_first = envelope.to[0].mailbox.to_s + '@' + envelope.to[0].host.to_s - mail_to_second = envelope.to[1].mailbox.to_s + '@' + envelope.to[1].host.to_s - if mail_to_first.casecmp(@mail_address).zero? || - (@aliases.any? { |s| s.casecmp(mail_to_first).zero? }) - helper = CollectorHelper.new( - envelope.from[0].mailbox.to_s + '@' + envelope.from[0].host.to_s, - mail_to_second - ) - elsif mail_to_second.casecmp(@mail_address).zero? || - (@aliases.any? { |s| s.casecmp(mail_to_second).zero? }) - helper = CollectorHelper.new( - envelope.from[0].mailbox.to_s + '@' + envelope.from[0].host.to_s, - mail_to_first - ) - end - elsif !envelope.cc && envelope.to && envelope.to.length == 1 - log_info 'Using Sender = Recipient method...' - mail_to = envelope.to[0].mailbox.to_s + '@' + envelope.to[0].host.to_s - if mail_to.casecmp(@mail_address).zero? || (@aliases.any? { |s| s.casecmp(mail_to).zero? }) - helper = CollectorHelper.new( - envelope.from[0].mailbox.to_s + '@' + envelope.from[0].host.to_s - ) - end - end - rescue => e - log_error 'Error on mailcollector create_helper:' - log_error e.backtrace.join('\n') - raise + def email_eln_email?(mail_to) + mail_to.casecmp(@mail_address).zero? || + (@aliases.any? { |s| s.casecmp(mail_to).zero? }) + end + + def get_user(mail) + user = User.find_by email: mail + user ||= User.find_by email: mail.downcase + user + end + + def create_helper_set(envelope) + # Check if from is User or Device + from = "#{envelope.from[0].mailbox}@#{envelope.from[0].host}" + from_user = get_user from + raise "#{from} not registered" if from_user.nil? + + receiver = [] + if from_user.is_a?(User) && (!from_user.is_a?(Device) && !from_user.is_a?(Admin)) + receiver.push(from_user) + else # Concatenate all receiver (cc & to) + receiver.concat(envelope.cc) if envelope.cc + receiver.concat(envelope.to) if envelope.to + receiver = receiver.map { |m| "#{m.mailbox}@#{m.host}" } + .reject { |m| email_eln_email?(m) } + .map { |m| get_user(m) } + .select { |user| !user.nil? && !user.is_a?(Device) && !user.is_a?(Admin) } end - helper + return nil if receiver.empty? + + CollectorHelperSet.new(from_user, receiver) + rescue StandardError => e + log_error 'Error on mailcollector create_helper:' + log_error e.backtrace.join('\n') + raise end def log_info(message) - DCLogger.log.info(self.class.name) { - ' >>> ' + message - } + DCLogger.log.info(self.class.name) do + " >>> #{message}" + end end def log_error(message) - DCLogger.log.error(self.class.name) { - ' >>> ' + message - } + DCLogger.log.error(self.class.name) do + " >>> #{message}" + end end end