diff --git a/py/desitransfer/spacewatch.py b/py/desitransfer/spacewatch.py index 39c905f..5b5bd1f 100644 --- a/py/desitransfer/spacewatch.py +++ b/py/desitransfer/spacewatch.py @@ -67,11 +67,18 @@ def jpg_list(index): A list of JPEG files found in `index`. The `index` URL is attached to the file names. """ - r = requests.get(index) - parser = SpacewatchHTMLParser() + try: + r = requests.get(index) + except (requests.RequestException, requests.ConnectionError, requests.HTTPError) as e: + log.critical(e.args[0]) + return [] if r.status_code == 200: + parser = SpacewatchHTMLParser() parser.feed(r.content.decode(r.headers['Content-Type'].split('=')[1])) - return [index + j for j in parser.jpg_files] + return [index + j for j in parser.jpg_files] + else: + log.critical("Unexpected status when listing JPEG files: %d!", r.status_code) + return [] def download_jpg(files, destination, overwrite=False, test=False): @@ -172,9 +179,9 @@ def main(): spacewatch_yesterday = spacewatch_root + ystrdy + '/' n_files = download_jpg(jpg_list(spacewatch_today), os.path.join(options.destination, today), overwrite=options.overwrite, test=options.test) - log.info("%d files downloaded for %s.", n_files, today) + log.debug("%d files downloaded for %s.", n_files, today) if options.date is None: n_files = download_jpg(jpg_list(spacewatch_yesterday), os.path.join(options.destination, ystrdy), overwrite=options.overwrite, test=options.test) - log.info("%d files downloaded for %s.", n_files, ystrdy) + log.debug("%d files downloaded for %s.", n_files, ystrdy) return 0 diff --git a/py/desitransfer/test/test_spacewatch.py b/py/desitransfer/test/test_spacewatch.py index 510310e..984f703 100644 --- a/py/desitransfer/test/test_spacewatch.py +++ b/py/desitransfer/test/test_spacewatch.py @@ -158,6 +158,38 @@ def test_jpg_files(self, mock_requests): 'http://foo.bar/20231031_000405.jpg', 'http://foo.bar/20231031_000605.jpg']) + @patch('desitransfer.spacewatch.log') + @patch('desitransfer.spacewatch.requests') + def test_jpg_files_bad_status(self, mock_requests, mock_log): + """Test bad HTTP status on jpg file list. + """ + mock_contents = Mock() + mock_contents.headers = {'Content-Type': 'text/html;charset=ISO-8859-1'} + mock_contents.status_code = 404 + mock_contents.content = b'' + mock_requests.get.return_value = mock_contents + jpg_files = jpg_list('http://foo.bar/') + self.assertListEqual(jpg_files, []) + mock_log.critical.assert_called_once_with("Unexpected status when listing JPEG files: %d!", 404) + + @patch('desitransfer.spacewatch.log') + @patch('desitransfer.spacewatch.requests') + def test_jpg_files_request_exception(self, mock_requests, mock_log): + """Test requests exceptions for jpg list. + """ + # mock_contents = Mock() + # mock_contents.headers = {'Content-Type': 'text/html;charset=ISO-8859-1'} + # mock_contents.status_code = 404 + # mock_contents.content = b'' + mock_requests.RequestException = Exception + mock_requests.ConnectionError = Exception + mock_requests.HTTPError = Exception + msg = "Exception thrown when attempting to access file list!" + mock_requests.get.side_effect = mock_requests.ConnectionError(msg) + jpg_files = jpg_list('http://foo.bar/') + self.assertListEqual(jpg_files, []) + mock_log.critical.assert_called_once_with(msg) + @patch('desitransfer.spacewatch.log') @patch('os.utime') @patch('desitransfer.spacewatch.requests')