From 88164760fa013f118ee0b3338d740b7282b48d2c Mon Sep 17 00:00:00 2001 From: clemensgg Date: Fri, 1 Sep 2023 13:48:24 +0200 Subject: [PATCH 1/3] extract cr data to tmp, increase cache timeout --- app.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index 7357ab4..58c40fd 100644 --- a/app.py +++ b/app.py @@ -11,6 +11,7 @@ import os import zipfile import json +import tempfile app = Flask(__name__) @@ -45,6 +46,9 @@ def download_and_extract_repo(): """Download the GitHub repository as a zip file, extract it, and return the path to the extracted content.""" global repo_path + # Create a temporary directory for extraction + temp_dir = tempfile.mkdtemp() + # GitHub API endpoint to get the zip download URL repo_api_url = "https://api.github.com/repos/cosmos/chain-registry" headers = { @@ -72,12 +76,14 @@ def download_and_extract_repo(): # Extract the zip file with zipfile.ZipFile(zip_filename, 'r') as zip_ref: - zip_ref.extractall('.') - # The extracted folder has a name like 'cosmos-chain-registry-' - # We'll return the first directory that starts with 'cosmos-chain-registry-' - extracted_folder = next((folder for folder in os.listdir('.') if folder.startswith('cosmos-chain-registry-')), None) - repo_path = extracted_folder - return repo_path + zip_ref.extractall(temp_dir) + extracted_folder = next((folder for folder in os.listdir(temp_dir) if folder.startswith('cosmos-chain-registry-')), None) + new_repo_path = os.path.join(temp_dir, extracted_folder) + + # Update the global repo_path only after successful extraction + repo_path = new_repo_path + + return repo_path def fetch_directory_names(path): headers = {'Accept': 'application/vnd.github.v3+json'} @@ -443,7 +449,7 @@ def fetch_network_data(): return jsonify({"error": str(e)}), 500 @app.route('/mainnets') -@cache.cached(timeout=300) # Cache the result for 5 minutes +@cache.cached(timeout=600) # Cache the result for 10 minutes def get_mainnet_data(): results = cache.get('MAINNET_DATA') if results is None: @@ -456,7 +462,7 @@ def get_mainnet_data(): return jsonify(sorted_results) @app.route('/testnets') -@cache.cached(timeout=300) # Cache the result for 5 minutes +@cache.cached(timeout=600) # Cache the result for 10 minutes def get_testnet_data(): results = cache.get('TESTNET_DATA') if results is None: From 5e0e3ac1354487ab74435108f35feaef74101b02 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Fri, 1 Sep 2023 13:54:15 +0200 Subject: [PATCH 2/3] add execiton time counter to `update_data` --- app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 58c40fd..c48a37a 100644 --- a/app.py +++ b/app.py @@ -371,6 +371,7 @@ def fetch_data_for_network(network, network_type): def update_data(): """Function to periodically update the data for mainnets and testnets.""" while True: + start_time = datetime.now() print("Starting data update cycle...") try: repo_path = download_and_extract_repo() @@ -395,10 +396,13 @@ def update_data(): cache.set('MAINNET_DATA', mainnet_data) cache.set('TESTNET_DATA', testnet_data) - print("Data update cycle completed. Sleeping for 1 minute...") + elapsed_time = (datetime.now() - start_time).total_seconds() + print(f"Data update cycle completed in {elapsed_time} seconds. Sleeping for 1 minute...") sleep(60) + except Exception as e: - print(f"Error in update_data loop: {e}") + elapsed_time = (datetime.now() - start_time).total_seconds() + print(f"Error in update_data loop after {elapsed_time} seconds: {e}") print("Error encountered. Sleeping for 1 minute before retrying...") sleep(60) From 335b8af12cd5a137a3666512c436264ac2585ae7 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Fri, 1 Sep 2023 14:06:13 +0200 Subject: [PATCH 3/3] retain cr repo for `REPO_RETAIN_HOURS` h --- app.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index c48a37a..3e5abe5 100644 --- a/app.py +++ b/app.py @@ -24,8 +24,10 @@ # Initialize cache cache = Cache(app, config={'CACHE_TYPE': 'simple'}) -# Initialize repo path var +# Initialize repo vars repo_path = "" +repo_last_download_time = None +repo_retain_hours = int(os.environ.get('REPO_RETAIN_HOURS', 3)) # Initialize number of workers num_workers = int(os.environ.get('NUM_WORKERS', 10)) @@ -370,13 +372,22 @@ def fetch_data_for_network(network, network_type): # periodic cache update def update_data(): """Function to periodically update the data for mainnets and testnets.""" + global repo_last_download_time + while True: - start_time = datetime.now() + start_time = datetime.now() # Capture the start time print("Starting data update cycle...") - try: - repo_path = download_and_extract_repo() - print(f"Repository downloaded and extracted to: {repo_path}") + # Check if it's time to download the repo + if repo_last_download_time is None or (datetime.now() - repo_last_download_time).total_seconds() >= 60 * 60 * repo_retain_hours: + try: + repo_path = download_and_extract_repo() + print(f"Repository downloaded and extracted to: {repo_path}") + repo_last_download_time = datetime.now() + except Exception as e: + print(f"Error downloading and extracting repo: {e}") + + try: # Process mainnets & testnets mainnet_networks = [d for d in os.listdir(repo_path) if os.path.isdir(os.path.join(repo_path, d)) @@ -396,16 +407,16 @@ def update_data(): cache.set('MAINNET_DATA', mainnet_data) cache.set('TESTNET_DATA', testnet_data) - elapsed_time = (datetime.now() - start_time).total_seconds() + elapsed_time = (datetime.now() - start_time).total_seconds() # Calculate the elapsed time print(f"Data update cycle completed in {elapsed_time} seconds. Sleeping for 1 minute...") sleep(60) - except Exception as e: - elapsed_time = (datetime.now() - start_time).total_seconds() + elapsed_time = (datetime.now() - start_time).total_seconds() # Calculate the elapsed time in case of an error print(f"Error in update_data loop after {elapsed_time} seconds: {e}") print("Error encountered. Sleeping for 1 minute before retrying...") sleep(60) + def start_update_data_thread(): print("Starting the update_data thread...") update_thread = threading.Thread(target=update_data)