diff --git a/setup.py b/setup.py index deca18c8..c4d7977c 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2015. +# Copyright (c) 2015, 2020. # # Author(s): diff --git a/trollmoves/movers.py b/trollmoves/movers.py index 5284c908..cc32ea46 100644 --- a/trollmoves/movers.py +++ b/trollmoves/movers.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 +# Copyright (c) 2012-2020 # # Author(s): # @@ -31,12 +31,13 @@ import traceback from ftplib import FTP, all_errors, error_perm from threading import Event, Lock, Thread, current_thread +import netrc from six import string_types from six.moves.urllib.parse import urlparse -from trollmoves.utils import clean_url +from trollmoves.utils import clean_url from paramiko import SSHClient, SSHException, AutoAddPolicy from scp import SCPClient @@ -59,8 +60,10 @@ def move_it(pathname, destination, attrs=None, hook=None, rel_path=None): new_dest = dest_url._replace(path=new_path) fake_dest = clean_url(new_dest) + LOGGER.debug("new_dest = %s", new_dest) LOGGER.debug("Copying to: %s", fake_dest) try: + LOGGER.debug("Scheme = %s", str(dest_url.scheme)) mover = MOVERS[dest_url.scheme] except KeyError: LOGGER.error("Unsupported protocol '" + str(dest_url.scheme) @@ -91,6 +94,10 @@ def __init__(self, origin, destination, attrs=None): else: self.destination = destination + self._dest_username = self.destination.username + self._dest_password = self.destination.password + + LOGGER.debug("Destination: %s", str(destination)) self.origin = origin self.attrs = attrs or {} @@ -105,9 +112,12 @@ def move(self): + " not implemented (yet).") def get_connection(self, hostname, port, username=None): + """Get the connection.""" with self.active_connection_lock: + LOGGER.debug("Destination username and passwd: %s %s", + self._dest_username, self._dest_password) LOGGER.debug('Getting connection to %s@%s:%s', - self.destination.username, self.destination.hostname, port) + username, hostname, port) try: connection, timer = self.active_connections[(hostname, port, username)] if not self.is_connected(connection): @@ -128,7 +138,7 @@ def get_connection(self, hostname, port, username=None): def delete_connection(self, connection): with self.active_connection_lock: LOGGER.debug('Closing connection to %s@%s:%s', - self.destination.username, self.destination.hostname, self.destination.port) + self._dest_username, self.destination.hostname, self.destination.port) try: if current_thread().finished.is_set(): return @@ -201,13 +211,36 @@ class FtpMover(Mover): active_connections = dict() active_connection_lock = Lock() + def _get_netrc_authentication(self): + """Get login authentications from netrc file if available""" + try: + secrets = netrc.netrc() + except (netrc.NetrcParseError, FileNotFoundError) as e__: + LOGGER.warning('Failed retrieve authentification details from netrc file! Exception: %s', str(e__)) + return + + LOGGER.debug("Destination hostname: %s", self.destination.hostname) + LOGGER.debug("hosts: %s", str(list(secrets.hosts.keys()))) + LOGGER.debug("Check if hostname matches any listed in the netrc file") + if self.destination.hostname in list(secrets.hosts.keys()): + self._dest_username, account, self._dest_password = secrets.authenticators(self.destination.hostname) + LOGGER.debug('Got username and password from netrc file!') + def open_connection(self): + """Open the connection and login.""" + connection = FTP(timeout=10) + LOGGER.debug("Connect...") connection.connect(self.destination.hostname, self.destination.port or 21) - if self.destination.username and self.destination.password: - connection.login(self.destination.username, - self.destination.password) + + if not self._dest_username or not self._dest_password: + # Check if usernams, password is stored in the $(HOME)/.netrc file: + self._get_netrc_authentication() + LOGGER.debug("Authentication retrieved from netrc file!") + + if self._dest_username and self._dest_password: + connection.login(self._dest_username, self._dest_password) else: connection.login() @@ -239,7 +272,7 @@ def move(self): def copy(self): """Push it ! """ - connection = self.get_connection(self.destination.hostname, self.destination.port, self.destination.username) + connection = self.get_connection(self.destination.hostname, self.destination.port, self._dest_username) def cd_tree(current_dir): if current_dir != "": @@ -275,13 +308,13 @@ def open_connection(self): ssh_connection.set_missing_host_key_policy(AutoAddPolicy()) ssh_connection.load_system_host_keys() ssh_connection.connect(self.destination.hostname, - username=self.destination.username, + username=self._dest_username, port=self.destination.port or 22, key_filename=ssh_key_filename) LOGGER.debug("Successfully connected to %s:%s as %s", self.destination.hostname, self.destination.port or 22, - self.destination.username) + self._dest_username) except SSHException as sshe: LOGGER.error("Failed to init SSHClient: %s", str(sshe)) except Exception as err: @@ -323,7 +356,7 @@ def copy(self): ssh_connection = self.get_connection(self.destination.hostname, self.destination.port or 22, - self.destination.username) + self._dest_username) try: scp = SCPClient(ssh_connection.get_transport()) @@ -385,7 +418,7 @@ def _agent_auth(self, transport): LOGGER.debug('Trying ssh key %s', key.get_fingerprint().encode('hex')) try: - transport.auth_publickey(self.destination.username, key) + transport.auth_publickey(self._dest_username, key) LOGGER.debug('... ssh key success!') return except paramiko.SSHException: diff --git a/trollmoves/tests/test_movers.py b/trollmoves/tests/test_movers.py new file mode 100644 index 00000000..1a3b1960 --- /dev/null +++ b/trollmoves/tests/test_movers.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2020 Pytroll + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of + +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test the movers +""" + +from unittest.mock import patch + + +@patch('netrc.netrc') +def test_open_connection(netrc): + """Check getting the netrc authentication.""" + from trollmoves.movers import FtpMover + + username = 'myusername' + password = 'mypasswd' + account = None + + origin = '/path/to/mydata/filename.ext' + + netrc.side_effect = FileNotFoundError('Failed retrieve authentification details from netrc file') + + with patch('trollmoves.movers.FTP') as mymock: + destination = 'ftp://localhost.smhi.se/data/satellite/archive/' + ftp_mover = FtpMover(origin, destination) + ftp_mover.open_connection() + + mymock.return_value.login.assert_called_once_with() + + netrc.return_value.hosts = {'localhost.smhi.se': ('myusername', None, 'mypasswd')} + netrc.return_value.authenticators.return_value = (username, account, password) + netrc.side_effect = None + + with patch('trollmoves.movers.FTP') as mymock: + destination = 'ftp://localhost.smhi.se/data/satellite/archive/' + ftp_mover = FtpMover(origin, destination) + ftp_mover.open_connection() + + mymock.return_value.login.assert_called_once_with(username, password) + + with patch('trollmoves.movers.FTP') as mymock: + destination = 'ftp://auser:apasswd@localhost.smhi.se/data/satellite/archive/' + ftp_mover = FtpMover(origin, destination) + ftp_mover.open_connection() + + mymock.return_value.login.assert_called_once_with('auser', 'apasswd')