[#56] WIP: refactor copa scrape_data_csv function to return response …

…object, copa scrape transformer add api error handling and update tests
invinst · Oct 1, 2019 · 6ae5fc7 · 6ae5fc7
1 parent b5eefea
commit 6ae5fc7
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 6 deletions.
diff --git a/invisible_flow/api/copa_scrape.py b/invisible_flow/api/copa_scrape.py
@@ -36,5 +36,4 @@ def scrape_copa_not_in_entity(self):
 
     def scrape_data_csv(self):
         url = SCRAPE_URL + ".csv"
-        resp = requests.get(url=url)
-        return resp.content
+        return requests.get(url=url)
diff --git a/invisible_flow/transformers/copa_scrape_transformer.py b/invisible_flow/transformers/copa_scrape_transformer.py
@@ -17,9 +17,14 @@ def __init__(self):
 
     def save_scraped_data(self):
         scraper = CopaScrape()
-        csv = scraper.scrape_data_csv()
-        self.storage.store_string('initial_data.csv', csv, f'Scrape-{self.current_date}/initial_data')
-        self.create_and_save_metadata('initial_data')
+        response = scraper.scrape_data_csv()
+        csv = response.content
+        if response.status_code == 200:
+            self.storage.store_string('initial_data.csv', csv, f'Scrape-{self.current_date}/initial_data')
+            self.create_and_save_metadata('initial_data')
+        else:
+            error_to_write = str(response.status_code) + "\n" + response.text
+            self.storage.store_string('transform_error.csv', error_to_write, f'Scrape-{self.current_date}/errors')
 
     def split(self) -> Dict[str, List]:
         return {

diff --git a/tests/api/test_copa_scrape.py b/tests/api/test_copa_scrape.py
@@ -51,7 +51,7 @@ def default_fixture(request):
 class TestCopaScrape():
     def test_scrape_data_csv(self, get_mock):
         should_be_bubbles = CopaScrape().scrape_data_csv()
-        assert should_be_bubbles == "bubbles"
+        assert should_be_bubbles.content == "bubbles"
 
     def test_scrape_data_json(self, get_mock):
         should_be_json_data = CopaScrape().scrape_data_json()

diff --git a/tests/transformers/test_copa_scrape_transformer.py b/tests/transformers/test_copa_scrape_transformer.py
@@ -21,6 +21,8 @@ def __init__(self, json_data, status_code, content):
             self.json_data = json_data
             self.status_code = status_code
             self.content = content
+            self.message = "DANGER DANGER"
+            self.text = "response text property value"
 
         def json(self):
             return self.json_data
@@ -92,3 +94,21 @@ def test_transform(self, get_mock):
             call("misc-data.csv", b"some content", f'Scrape-{self.current_date}/transformed')
         ]
         store_string_mock.assert_has_calls(calls)
+
+    @patch('invisible_flow.app.GlobalsFactory.get_current_datetime_utc', lambda: datetime(2019, 3, 25, 5, 30, 50, 0))
+    def test_save_scraped_data_with_all_response_codes(self, get_mock):
+        if response_code == 200:
+            filename = "initial_data.csv"
+            pathname = "initial_data"
+        else:
+            filename = "transform_error.csv"
+            pathname = "errors"
+        with patch('invisible_flow.app.StorageFactory.get_storage') as get_storage_mock:
+            with patch('invisible_flow.storage.LocalStorage.store_string') as store_string_mock:
+                get_storage_mock.return_value = LocalStorage()
+                CopaScrapeTransformer().save_scraped_data()
+                self.current_date = GlobalsFactory.get_current_datetime_utc().isoformat(sep='_').replace(':', '-')
+                calls = [
+                    call(filename, mock.ANY, f'Scrape-{self.current_date}/{pathname}')
+                ]
+                store_string_mock.assert_has_calls(calls)