Changes in run_data_collection.py to examine if issue #59 still occurrs

nickpadd · Nov 30, 2023 · e4baaa9 · e4baaa9
1 parent 58d5a0c
commit e4baaa9
Showing 1 changed file with 6 additions and 9 deletions.
diff --git a/run_data_collection.py b/run_data_collection.py
@@ -9,25 +9,23 @@
 import pandas as pd
 from europeanfootballleaguepredictor.data.database_handler import DatabaseHandler 
 import aiohttp 
+import time
 from aiohttp.client_exceptions import ServerDisconnectedError
 import sys
 
-async def fetch_data_with_retry(understat_parser, season, months_of_form, output_table_name, max_retries=3, retry_delay=1):
+def fetch_data_with_retry(understat_parser, season, months_of_form, output_table_name, max_retries=3, retry_delay=1):
     for attempt in range(max_retries):
         try:
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
-            future = asyncio.run_coroutine_threadsafe(
-                understat_parser.get_understat_season(season=season, months_of_form=months_of_form, output_table_name=output_table_name),
-                loop
-            )
-            future.result()  # Wait for the result
+            loop.run_until_complete(understat_parser.get_understat_season(season=season, months_of_form=months_of_form, output_table_name=output_table_name))
+            time.sleep(1)
             logger.success(f'Succesfully gathered and saved season {season}.')
             break  # Break out of the loop if successful
         except (aiohttp.ClientError, ServerDisconnectedError) as e:
             if attempt < max_retries - 1:
                 print(f"Retry attempt {attempt + 1}/{max_retries}")
-                await asyncio.sleep(retry_delay)  # Add a short delay before retrying
+                time.sleep(retry_delay)  # Add a short delay before retrying
             else:
                 logger.error(f'Max retries reached, for error {e}.')
                 sys.exit(1)
@@ -71,8 +69,7 @@ def main():
     for table_name, months_of_form in zip(['Raw_LongTermForm', 'Raw_ShortTermForm'], config.months_of_form_list):
         logger.info(f'Gathering {months_of_form} month form data for seasons in {config.seasons_to_gather}')
         for season in config.seasons_to_gather:
-            asyncio.run(fetch_data_with_retry(understat_parser, season, months_of_form, table_name))
-            asyncio.sleep(1)
+            fetch_data_with_retry(understat_parser, season, months_of_form, table_name)
 
     preprocessor = Preprocessor(league=config.league, database=config.database)