catch requests errors and quiet some outputs

py/desitransfer/spacewatch.py

@@ -67,11 +67,18 @@ def jpg_list(index):
         A list of JPEG files found in `index`. The `index` URL is attached
         to the file names.
     """
-    r = requests.get(index)
-    parser = SpacewatchHTMLParser()
+    try:
+        r = requests.get(index)
+    except (requests.RequestException, requests.ConnectionError, requests.HTTPError) as e:
+        log.critical(e.args[0])
+        return []
     if r.status_code == 200:
+        parser = SpacewatchHTMLParser()
         parser.feed(r.content.decode(r.headers['Content-Type'].split('=')[1]))
-    return [index + j for j in parser.jpg_files]
+        return [index + j for j in parser.jpg_files]
+    else:
+        log.critical("Unexpected status when listing JPEG files: %d!", r.status_code)
+        return []
 
 
 def download_jpg(files, destination, overwrite=False, test=False):
@@ -172,9 +179,9 @@ def main():
     spacewatch_yesterday = spacewatch_root + ystrdy + '/'
     n_files = download_jpg(jpg_list(spacewatch_today), os.path.join(options.destination, today),
                            overwrite=options.overwrite, test=options.test)
-    log.info("%d files downloaded for %s.", n_files, today)
+    log.debug("%d files downloaded for %s.", n_files, today)
     if options.date is None:
         n_files = download_jpg(jpg_list(spacewatch_yesterday), os.path.join(options.destination, ystrdy),
                                overwrite=options.overwrite, test=options.test)
-        log.info("%d files downloaded for %s.", n_files, ystrdy)
+        log.debug("%d files downloaded for %s.", n_files, ystrdy)
     return 0

py/desitransfer/test/test_spacewatch.py

@@ -158,6 +158,38 @@ def test_jpg_files(self, mock_requests):
                                          'http://foo.bar/20231031_000405.jpg',
                                          'http://foo.bar/20231031_000605.jpg'])
 
+    @patch('desitransfer.spacewatch.log')
+    @patch('desitransfer.spacewatch.requests')
+    def test_jpg_files_bad_status(self, mock_requests, mock_log):
+        """Test bad HTTP status on jpg file list.
+        """
+        mock_contents = Mock()
+        mock_contents.headers = {'Content-Type': 'text/html;charset=ISO-8859-1'}
+        mock_contents.status_code = 404
+        mock_contents.content = b''
+        mock_requests.get.return_value = mock_contents
+        jpg_files = jpg_list('http://foo.bar/')
+        self.assertListEqual(jpg_files, [])
+        mock_log.critical.assert_called_once_with("Unexpected status when listing JPEG files: %d!", 404)
+
+    @patch('desitransfer.spacewatch.log')
+    @patch('desitransfer.spacewatch.requests')
+    def test_jpg_files_request_exception(self, mock_requests, mock_log):
+        """Test requests exceptions for jpg list.
+        """
+        # mock_contents = Mock()
+        # mock_contents.headers = {'Content-Type': 'text/html;charset=ISO-8859-1'}
+        # mock_contents.status_code = 404
+        # mock_contents.content = b''
+        mock_requests.RequestException = Exception
+        mock_requests.ConnectionError = Exception
+        mock_requests.HTTPError = Exception
+        msg = "Exception thrown when attempting to access file list!"
+        mock_requests.get.side_effect = mock_requests.ConnectionError(msg)
+        jpg_files = jpg_list('http://foo.bar/')
+        self.assertListEqual(jpg_files, [])
+        mock_log.critical.assert_called_once_with(msg)
+
     @patch('desitransfer.spacewatch.log')
     @patch('os.utime')
     @patch('desitransfer.spacewatch.requests')