Merge pull request #78 from adybbroe/feature-netrc-authentication

Get username and password from .netrc file if available

setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2015.
+# Copyright (c) 2015, 2020.
 #
 
 # Author(s):

trollmoves/movers.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2012-2019
+# Copyright (c) 2012-2020
 #
 # Author(s):
 #
@@ -31,12 +31,13 @@
 import traceback
 from ftplib import FTP, all_errors, error_perm
 from threading import Event, Lock, Thread, current_thread
+import netrc
 
 from six import string_types
 from six.moves.urllib.parse import urlparse
 
-from trollmoves.utils import clean_url
 
+from trollmoves.utils import clean_url
 from paramiko import SSHClient, SSHException, AutoAddPolicy
 from scp import SCPClient
 
@@ -59,8 +60,10 @@ def move_it(pathname, destination, attrs=None, hook=None, rel_path=None):
     new_dest = dest_url._replace(path=new_path)
     fake_dest = clean_url(new_dest)
 
+    LOGGER.debug("new_dest = %s", new_dest)
     LOGGER.debug("Copying to: %s", fake_dest)
     try:
+        LOGGER.debug("Scheme = %s", str(dest_url.scheme))
         mover = MOVERS[dest_url.scheme]
     except KeyError:
         LOGGER.error("Unsupported protocol '" + str(dest_url.scheme)
@@ -91,6 +94,10 @@ def __init__(self, origin, destination, attrs=None):
         else:
             self.destination = destination
 
+        self._dest_username = self.destination.username
+        self._dest_password = self.destination.password
+
+        LOGGER.debug("Destination: %s", str(destination))
         self.origin = origin
         self.attrs = attrs or {}
 
@@ -105,9 +112,12 @@ def move(self):
                                   + " not implemented (yet).")
 
     def get_connection(self, hostname, port, username=None):
+        """Get the connection."""
         with self.active_connection_lock:
+            LOGGER.debug("Destination username and passwd: %s %s",
+                         self._dest_username, self._dest_password)
             LOGGER.debug('Getting connection to %s@%s:%s',
-                         self.destination.username, self.destination.hostname, port)
+                         username, hostname, port)
             try:
                 connection, timer = self.active_connections[(hostname, port, username)]
                 if not self.is_connected(connection):
@@ -128,7 +138,7 @@ def get_connection(self, hostname, port, username=None):
     def delete_connection(self, connection):
         with self.active_connection_lock:
             LOGGER.debug('Closing connection to %s@%s:%s',
-                         self.destination.username, self.destination.hostname, self.destination.port)
+                         self._dest_username, self.destination.hostname, self.destination.port)
             try:
                 if current_thread().finished.is_set():
                     return
@@ -201,13 +211,36 @@ class FtpMover(Mover):
     active_connections = dict()
     active_connection_lock = Lock()
 
+    def _get_netrc_authentication(self):
+        """Get login authentications from netrc file if available"""
+        try:
+            secrets = netrc.netrc()
+        except (netrc.NetrcParseError, FileNotFoundError) as e__:
+            LOGGER.warning('Failed retrieve authentification details from netrc file! Exception: %s', str(e__))
+            return
+
+        LOGGER.debug("Destination hostname: %s", self.destination.hostname)
+        LOGGER.debug("hosts: %s", str(list(secrets.hosts.keys())))
+        LOGGER.debug("Check if hostname matches any listed in the netrc file")
+        if self.destination.hostname in list(secrets.hosts.keys()):
+            self._dest_username, account, self._dest_password = secrets.authenticators(self.destination.hostname)
+            LOGGER.debug('Got username and password from netrc file!')
+
     def open_connection(self):
+        """Open the connection and login."""
+
         connection = FTP(timeout=10)
+        LOGGER.debug("Connect...")
         connection.connect(self.destination.hostname,
                            self.destination.port or 21)
-        if self.destination.username and self.destination.password:
-            connection.login(self.destination.username,
-                             self.destination.password)
+
+        if not self._dest_username or not self._dest_password:
+            # Check if usernams, password is stored in the $(HOME)/.netrc file:
+            self._get_netrc_authentication()
+            LOGGER.debug("Authentication retrieved from netrc file!")
+
+        if self._dest_username and self._dest_password:
+            connection.login(self._dest_username, self._dest_password)
         else:
             connection.login()
 
@@ -239,7 +272,7 @@ def move(self):
     def copy(self):
         """Push it !
         """
-        connection = self.get_connection(self.destination.hostname, self.destination.port, self.destination.username)
+        connection = self.get_connection(self.destination.hostname, self.destination.port, self._dest_username)
 
         def cd_tree(current_dir):
             if current_dir != "":
@@ -275,13 +308,13 @@ def open_connection(self):
                 ssh_connection.set_missing_host_key_policy(AutoAddPolicy())
                 ssh_connection.load_system_host_keys()
                 ssh_connection.connect(self.destination.hostname,
-                                       username=self.destination.username,
+                                       username=self._dest_username,
                                        port=self.destination.port or 22,
                                        key_filename=ssh_key_filename)
                 LOGGER.debug("Successfully connected to %s:%s as %s",
                              self.destination.hostname,
                              self.destination.port or 22,
-                             self.destination.username)
+                             self._dest_username)
             except SSHException as sshe:
                 LOGGER.error("Failed to init SSHClient: %s", str(sshe))
             except Exception as err:
@@ -323,7 +356,7 @@ def copy(self):
 
         ssh_connection = self.get_connection(self.destination.hostname,
                                              self.destination.port or 22,
-                                             self.destination.username)
+                                             self._dest_username)
 
         try:
             scp = SCPClient(ssh_connection.get_transport())
@@ -385,7 +418,7 @@ def _agent_auth(self, transport):
             LOGGER.debug('Trying ssh key %s',
                          key.get_fingerprint().encode('hex'))
             try:
-                transport.auth_publickey(self.destination.username, key)
+                transport.auth_publickey(self._dest_username, key)
                 LOGGER.debug('... ssh key success!')
                 return
             except paramiko.SSHException:

trollmoves/tests/test_movers.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2020 Pytroll
+
+# Author(s):
+
+#   Adam.Dybbroe <[email protected]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the movers
+"""
+
+from unittest.mock import patch
+
+
+@patch('netrc.netrc')
+def test_open_connection(netrc):
+    """Check getting the netrc authentication."""
+    from trollmoves.movers import FtpMover
+
+    username = 'myusername'
+    password = 'mypasswd'
+    account = None
+
+    origin = '/path/to/mydata/filename.ext'
+
+    netrc.side_effect = FileNotFoundError('Failed retrieve authentification details from netrc file')
+
+    with patch('trollmoves.movers.FTP') as mymock:
+        destination = 'ftp://localhost.smhi.se/data/satellite/archive/'
+        ftp_mover = FtpMover(origin, destination)
+        ftp_mover.open_connection()
+
+        mymock.return_value.login.assert_called_once_with()
+
+    netrc.return_value.hosts = {'localhost.smhi.se': ('myusername', None, 'mypasswd')}
+    netrc.return_value.authenticators.return_value = (username, account, password)
+    netrc.side_effect = None
+
+    with patch('trollmoves.movers.FTP') as mymock:
+        destination = 'ftp://localhost.smhi.se/data/satellite/archive/'
+        ftp_mover = FtpMover(origin, destination)
+        ftp_mover.open_connection()
+
+        mymock.return_value.login.assert_called_once_with(username, password)
+
+    with patch('trollmoves.movers.FTP') as mymock:
+        destination = 'ftp://auser:[email protected]/data/satellite/archive/'
+        ftp_mover = FtpMover(origin, destination)
+        ftp_mover.open_connection()
+
+        mymock.return_value.login.assert_called_once_with('auser', 'apasswd')